diff options
author | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:28:14 +0200 |
---|---|---|
committer | Øyvind Grønnesby <oyving@yahooinc.com> | 2022-05-30 12:28:14 +0200 |
commit | 9fa7903909043d7b855f7e3ba315050ba5a12597 (patch) | |
tree | 3b6654d0725d64ac27f92ae089f0897895b7059a | |
parent | 45d46ce6c3b3d8c85f1ec9cfbd3793a161571265 (diff) | |
parent | e657c0a9618868c9dcf32cfa7e05ac73750b904c (diff) |
Merge remote-tracking branch 'origin/master' into ogronnesby/contact-info-resources
1978 files changed, 9516 insertions, 14292 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 0eb126901b9..c0f68a1c0b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -80,7 +80,6 @@ add_subdirectory(docprocs) add_subdirectory(document) add_subdirectory(documentapi) add_subdirectory(eval) -add_subdirectory(fastlib) add_subdirectory(fastos) add_subdirectory(fbench) add_subdirectory(fileacquirer) @@ -95,7 +94,6 @@ add_subdirectory(jdisc_core) add_subdirectory(jdisc-security-filters) add_subdirectory(jdisc_jetty) add_subdirectory(jrt_test) -add_subdirectory(juniper) add_subdirectory(linguistics) add_subdirectory(linguistics-components) add_subdirectory(logd) @@ -112,22 +110,16 @@ add_subdirectory(node-admin) add_subdirectory(node-repository) add_subdirectory(orchestrator) add_subdirectory(persistence) -add_subdirectory(persistencetypes) add_subdirectory(predicate-search) -add_subdirectory(searchcommon) add_subdirectory(searchcore) -add_subdirectory(searchcorespi) add_subdirectory(searchlib) add_subdirectory(searchsummary) add_subdirectory(security-tools) add_subdirectory(security-utils) add_subdirectory(service-monitor) add_subdirectory(slobrok) -add_subdirectory(staging_vespalib) add_subdirectory(standalone-container) add_subdirectory(storage) -add_subdirectory(storageapi) -add_subdirectory(storageframework) add_subdirectory(storageserver) add_subdirectory(statistics) add_subdirectory(streamingvisitors) @@ -152,7 +144,6 @@ add_subdirectory(vespajlib) add_subdirectory(vespalib) add_subdirectory(vespalog) add_subdirectory(vespamalloc) -add_subdirectory(vsm) add_subdirectory(zkfacade) add_subdirectory(zookeeper-command-line-client) add_subdirectory(zookeeper-server) diff --git a/Code-map.md b/Code-map.md index 247828f4e77..40307ef7e30 100644 --- a/Code-map.md +++ b/Code-map.md @@ -93,7 +93,6 @@ It is implemented in Java. Libraries used throughput the code. - [vespalib](https://github.com/vespa-engine/vespa/tree/master/vespalib) - general utility library for C++ -- [staging_vespalib](https://github.com/vespa-engine/vespa/tree/master/staging_vespalib) - general (less stable) utility library for C++ - [vespajlib](https://github.com/vespa-engine/vespa/tree/master/vespajlib) - general utility library for Java. Includes the Java implementation of the tensor library. - [yolean](https://github.com/vespa-engine/vespa/tree/master/yolean) - another (more stable) general Java utility library diff --git a/application-model/CMakeLists.txt b/application-model/CMakeLists.txt index 3bc7d65bd67..df7fff3a202 100644 --- a/application-model/CMakeLists.txt +++ b/application-model/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(application-model) +install_jar(application-model-jar-with-dependencies.jar) diff --git a/application-preprocessor/CMakeLists.txt b/application-preprocessor/CMakeLists.txt index 479f0c64350..d4b0b766fd8 100644 --- a/application-preprocessor/CMakeLists.txt +++ b/application-preprocessor/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(application-preprocessor) +install_jar(application-preprocessor-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-preprocess-application bin) diff --git a/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java b/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java index 1b2226c7a17..90f87d0b37e 100644 --- a/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java +++ b/application/src/main/java/com/yahoo/application/container/DocumentAccesses.java @@ -5,7 +5,7 @@ import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.DocumentAccess; import com.yahoo.documentapi.DocumentAccessParams; import com.yahoo.documentapi.local.LocalDocumentAccess; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import java.io.File; import java.util.stream.Stream; diff --git a/athenz-identity-provider-service/CMakeLists.txt b/athenz-identity-provider-service/CMakeLists.txt index 0e9903dcf63..75208c49bd3 100644 --- a/athenz-identity-provider-service/CMakeLists.txt +++ b/athenz-identity-provider-service/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(athenz-identity-provider-service) +install_jar(athenz-identity-provider-service-jar-with-dependencies.jar) diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java index 518f65ded17..6b9b0845328 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeClassTest.java @@ -38,23 +38,23 @@ import static org.junit.Assert.fail; public class AnalyzeClassTest { @Test - public void require_that_full_class_name_is_returned() { + public void full_class_name_is_returned() { assertEquals(name(Base.class), analyzeClass(Base.class).getName()); } @Test - public void require_that_base_class_is_included() { + public void base_class_is_included() { assertTrue(analyzeClass(Derived.class).getReferencedClasses().contains(name(Base.class))); } @Test - public void require_that_implemented_interfaces_are_included() { + public void implemented_interfaces_are_included() { assertTrue(analyzeClass(Base.class).getReferencedClasses().containsAll( List.of(name(Interface1.class), name(Interface2.class)))); } @Test - public void require_that_interface_can_be_analyzed() { + public void interface_can_be_analyzed() { ClassFileMetaData classMetaData = analyzeClass(Interface1.class); assertEquals(name(Interface1.class), classMetaData.getName()); @@ -62,72 +62,72 @@ public class AnalyzeClassTest { } @Test - public void require_that_return_type_is_included() { + public void return_type_is_included() { assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(Image.class))); } @Test - public void require_that_parameters_are_included() { + public void parameters_are_included() { assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(Kernel.class))); } @Test - public void require_that_exceptions_are_included() { + public void exceptions_are_included() { assertTrue(analyzeClass(Interface1.class).getReferencedClasses().contains(name(ImagingOpException.class))); } @Test - public void require_that_basic_types_ignored() { + public void basic_types_ignored() { List.of("int", "float").forEach(type -> assertFalse(analyzeClass(Interface1.class).getReferencedClasses().contains(type))); } @Test - public void require_that_arrays_of_basic_types_ignored() { + public void arrays_of_basic_types_ignored() { List.of("int[]", "int[][]").forEach(type -> assertFalse(analyzeClass(Interface1.class).getReferencedClasses().contains(type))); } @Test - public void require_that_instance_field_types_are_included() { + public void instance_field_types_are_included() { assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(String.class))); } @Test - public void require_that_static_field_types_are_included() { + public void static_field_types_are_included() { assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(java.util.List.class))); } @Test - public void require_that_field_annotation_is_included() { + public void field_annotation_is_included() { assertTrue(analyzeClass(Fields.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test - public void require_that_class_annotation_is_included() { + public void class_annotation_is_included() { assertTrue(analyzeClass(ClassAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test - public void require_that_method_annotation_is_included() { + public void method_annotation_is_included() { assertTrue(analyzeClass(MethodAnnotation.class).getReferencedClasses().contains(name(DummyAnnotation.class))); } @Test - public void require_that_export_package_annotations_are_ignored() { + public void export_package_annotations_are_ignored() { List.of(ExportPackage.class, Version.class).forEach(type -> assertFalse(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")) .getReferencedClasses().contains(type))); } @Test - public void require_that_export_annotations_are_processed() { + public void export_annotations_are_processed() { assertEquals(Optional.of(new ExportPackageAnnotation(3, 1, 4, "TEST_QUALIFIER-2")), Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.package-info")).getExportPackage()); } @Test - public void require_that_export_annotations_are_validated() { + public void export_annotations_are_validated() { try { Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.invalid.package-info")); @@ -140,24 +140,24 @@ public class AnalyzeClassTest { } @Test - public void require_that_catch_clauses_are_included() { + public void catch_clauses_are_included() { assertTrue(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.CatchException")) .getReferencedClasses().contains(name(LoginException.class))); } @Test - public void require_that_class_references_are_included() { + public void class_references_are_included() { assertTrue(Analyze.analyzeClass(classFile("com.yahoo.container.plugin.classanalysis.sampleclasses.ClassReference")) .getReferencedClasses().contains(name(Interface1.class))); } @Test - public void require_that_return_type_of_method_invocations_are_included() { + public void return_type_of_method_invocations_are_included() { assertTrue(analyzeClass(MethodInvocation.class).getReferencedClasses().contains(name(Image.class))); } @Test - public void require_that_attributes_are_included() { + public void attributes_are_included() { //Uses com/coremedia/iso/Utf8.class from com.googlecode.mp4parser:isoparser:1.0-RC-1 assertTrue(Analyze.analyzeClass(classFile("class/Utf8")).getReferencedClasses() .contains("org.aspectj.weaver.MethodDeclarationLineNumber")); diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java index 1bb6cb8976e..0076869eaa5 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/AnalyzeMethodBodyTest.java @@ -27,52 +27,52 @@ import static org.junit.Assert.assertTrue; public class AnalyzeMethodBodyTest { @Test - public void require_that_class_of_locals_are_included() { + public void class_of_locals_are_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Base.class))); } @Test - public void require_that_class_of_locals_in_static_method_are_included() { + public void class_of_locals_in_static_method_are_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Derived.class))); } @Test - public void require_that_field_references_are_included() { + public void field_references_are_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().containsAll(List.of(name(java.util.List.class), name(Fields.class)))); } @Test - public void require_that_class_owning_field_is_included() { + public void class_owning_field_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(System.class))); } @Test - public void require_that_class_containing_method_is_included() { + public void class_containing_method_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(PrintStream.class))); } @Test - public void require_that_element_of_new_multidimensional_array_is_included() { + public void element_of_new_multidimensional_array_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Interface1.class))); } @Test - public void require_that_basic_arrays_are_not_included() { + public void basic_arrays_are_not_included() { assertFalse(analyzeClass(Methods.class).getReferencedClasses().contains("int[]")); } @Test - public void require_that_container_generic_parameters_are_included() { + public void container_generic_parameters_are_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Dummy.class))); } @Test - public void require_that_functional_interface_usage_is_included() { + public void functional_interface_usage_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(Interface3.class))); } @Test - public void require_that_class_owning_method_handler_is_included() { + public void class_owning_method_handler_is_included() { assertTrue(analyzeClass(Methods.class).getReferencedClasses().contains(name(ClassWithMethod.class))); } } diff --git a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java index 5bef1457c5f..31abe785cbe 100644 --- a/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java +++ b/bundle-plugin/src/test/java/com/yahoo/container/plugin/classanalysis/PackageTallyTest.java @@ -14,7 +14,7 @@ import static org.junit.Assert.assertEquals; public class PackageTallyTest { @Test - public void referenced_packages_missing_from_are_detected() { + public void referenced_packages_missing_from_available_packages_are_detected() { PackageTally tally = new PackageTally(Map.of(), Set.of("p1", "java.util", "com.yahoo.api.annotations", "missing")); Set<String> missingPackages = tally.referencedPackagesMissingFrom(Set.of("p1")); assertEquals(Set.of("missing"), missingPackages); diff --git a/cloud-tenant-base-dependencies-enforcer/pom.xml b/cloud-tenant-base-dependencies-enforcer/pom.xml index b8c08843c9a..c44915c7ade 100644 --- a/cloud-tenant-base-dependencies-enforcer/pom.xml +++ b/cloud-tenant-base-dependencies-enforcer/pom.xml @@ -16,11 +16,20 @@ <version>7-SNAPSHOT</version> <packaging>pom</packaging> - <!-- MUST BE KEPT IN SYNC WITH container-dependency-versions pom - Copied here because vz-tenant-base does not have a parent. --> <properties> - <aopalliance.version>1.0</aopalliance.version> + <junit5.platform.version>1.8.1</junit5.platform.version> + + <!-- MUST BE KEPT IN SYNC WITH parent/pom.xml --> <athenz.version>1.10.14</athenz.version> + <httpclient5.version>5.1.2</httpclient5.version> + <junit5.version>5.8.1</junit5.version> <!-- TODO: in parent this is named 'junit.version' --> + <onnxruntime.version>1.8.0</onnxruntime.version> + <!-- END parent/pom.xml --> + + + <!-- ALL BELOW MUST BE KEPT IN SYNC WITH container-dependency-versions pom + Copied here because vz-tenant-base does not have a parent. --> + <aopalliance.version>1.0</aopalliance.version> <bouncycastle.version>1.68</bouncycastle.version> <felix.version>7.0.1</felix.version> <felix.log.version>1.0.1</felix.log.version> @@ -32,17 +41,13 @@ <jaxb.version>2.3.0</jaxb.version> <jetty.version>9.4.46.v20220331</jetty.version> <jetty-alpn.version>1.1.3.v20160715</jetty-alpn.version> - <junit5.version>5.8.1</junit5.version> - <junit5.platform.version>1.8.1</junit5.platform.version> - <onnxruntime.version>1.8.0</onnxruntime.version> <org.lz4.version>1.8.0</org.lz4.version> <org.json.version>20090211</org.json.version> - <slf4j.version>1.7.32</slf4j.version> + <slf4j.version>1.7.32</slf4j.version> <!-- WARNING: when updated, also update c.y.v.tenant:base pom --> <xml-apis.version>1.4.01</xml-apis.version> <hk2.version>2.5.0-b32</hk2.version> <hk2.osgi-resource-locator.version>1.0.1</hk2.osgi-resource-locator.version> - <httpclient5.version>5.1.2</httpclient5.version> <jackson2.version>2.12.6</jackson2.version> <jackson-databind.version>${jackson2.version}</jackson-databind.version> <javassist.version>3.20.0-GA</javassist.version> @@ -81,14 +86,15 @@ <include>com.fasterxml.jackson.datatype:jackson-datatype-jdk8:[${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.datatype:jackson-datatype-jsr310:[${jackson2.version}]:jar:provided</include> - <!-- Use version range for jax deps, because jersey and junit affect the versions. --> <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-base:[2.5.4, ${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:[2.5.4, ${jackson2.version}]:jar:provided</include> <include>com.fasterxml.jackson.module:jackson-module-jaxb-annotations:[2.5.4, ${jackson2.version}]:jar:provided</include> + <!-- guava with dependencies --> <include>com.google.code.findbugs:jsr305:[${findbugs.version}]:jar:provided</include> <include>com.google.guava:guava:[${guava.version}]:jar:provided</include> + <include>com.google.inject.extensions:guice-assistedinject:[${guice.version}]:jar:provided</include> <include>com.google.inject.extensions:guice-multibindings:[${guice.version}]:jar:provided</include> <include>com.google.inject:guice:[${guice.version}]:jar:provided:no_aop</include> @@ -252,6 +258,7 @@ <include>org.eclipse.jetty.http2:http2-server:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-alpn-server:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-alpn-java-server:[${jetty.version}]:jar:test</include> + <include>org.eclipse.jetty:jetty-client:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-continuation:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-jmx:[${jetty.version}]:jar:test</include> <include>org.eclipse.jetty:jetty-security:[${jetty.version}]:jar:test</include> diff --git a/cloud-tenant-cd/CMakeLists.txt b/cloud-tenant-cd/CMakeLists.txt index e9498d21871..74ce3f5d7ec 100644 --- a/cloud-tenant-cd/CMakeLists.txt +++ b/cloud-tenant-cd/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(cloud-tenant-cd) +install_jar(cloud-tenant-cd-jar-with-dependencies.jar) diff --git a/clustercontroller-apps/CMakeLists.txt b/clustercontroller-apps/CMakeLists.txt index d18180fabee..54918159ace 100644 --- a/clustercontroller-apps/CMakeLists.txt +++ b/clustercontroller-apps/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(clustercontroller-apps) +install_jar(clustercontroller-apps-jar-with-dependencies.jar) diff --git a/clustercontroller-core/CMakeLists.txt b/clustercontroller-core/CMakeLists.txt index bd0d8b26cf4..c7d127b4d0a 100644 --- a/clustercontroller-core/CMakeLists.txt +++ b/clustercontroller-core/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(clustercontroller-core) +install_jar(clustercontroller-core-jar-with-dependencies.jar) diff --git a/clustercontroller-reindexer/CMakeLists.txt b/clustercontroller-reindexer/CMakeLists.txt index d2407767e0d..436184268b0 100644 --- a/clustercontroller-reindexer/CMakeLists.txt +++ b/clustercontroller-reindexer/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(clustercontroller-reindexer) +install_jar(clustercontroller-reindexer-jar-with-dependencies.jar) diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java index 5737f038a17..1bab8f6ee27 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexerTest.java @@ -14,7 +14,7 @@ import com.yahoo.documentapi.VisitorParameters; import com.yahoo.documentapi.messagebus.protocol.DocumentProtocol; import com.yahoo.jdisc.test.MockMetric; import com.yahoo.messagebus.DynamicThrottlePolicy; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.test.ManualClock; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.jupiter.api.BeforeEach; @@ -31,7 +31,6 @@ import java.util.concurrent.Executor; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; -import java.util.stream.Collectors; import static java.util.stream.Collectors.toList; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java index c5212bdbfff..60dd0a1589d 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingCuratorTest.java @@ -5,7 +5,7 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.ProgressToken; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.jupiter.api.Test; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java index c5043f0fac3..e92d1e2ce1a 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/ReindexingMaintainerTest.java @@ -6,7 +6,7 @@ import ai.vespa.reindexing.Reindexing.Trigger; import com.yahoo.cloud.config.ClusterListConfig; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.vespa.config.content.AllClustersBucketSpacesConfig; import com.yahoo.vespa.config.content.reindexing.ReindexingConfig; import org.junit.jupiter.api.Test; diff --git a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java index 6ab41f4a7f7..2e3f4942b7d 100644 --- a/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java +++ b/clustercontroller-reindexer/src/test/java/ai/vespa/reindexing/http/ReindexingV1ApiTest.java @@ -10,7 +10,7 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.documentapi.ProgressToken; import com.yahoo.jdisc.test.MockMetric; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.vespa.curator.mock.MockCurator; import org.junit.jupiter.api.Test; diff --git a/clustercontroller-utils/CMakeLists.txt b/clustercontroller-utils/CMakeLists.txt index 53f28f7c1ab..ae5dc312869 100644 --- a/clustercontroller-utils/CMakeLists.txt +++ b/clustercontroller-utils/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(clustercontroller-utils) +install_jar(clustercontroller-utils-jar-with-dependencies.jar) diff --git a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java index 793020ce104..9e72efab07b 100644 --- a/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java +++ b/config-application-package/src/main/java/com/yahoo/config/model/application/provider/FilesApplicationPackage.java @@ -788,11 +788,11 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { Map.entry(QUERY_PROFILE_TYPES_DIR, Set.of(".xml")), Map.entry(Path.fromString(ROUTINGTABLES_DIR), Set.of(".xml")), Map.entry(RULES_DIR, Set.of(RULES_NAME_SUFFIX)), - // Note: Might have rank profiles in subdirs: schema-name]/[rank-profile].profile - Map.entry(SCHEMAS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)), + // Note: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile + Map.entry(SCHEMAS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX, RANKPROFILE_NAME_SUFFIX)), Map.entry(Path.fromString(SEARCHCHAINS_DIR), Set.of(".xml")), // Note: Might have rank profiles in subdirs: [schema-name]/[rank-profile].profile - Map.entry(SEARCH_DEFINITIONS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX)), + Map.entry(SEARCH_DEFINITIONS_DIR, Set.of(SD_NAME_SUFFIX, RANKEXPRESSION_NAME_SUFFIX, RANKPROFILE_NAME_SUFFIX)), Map.entry(SECURITY_DIR, Set.of(".pem"))); } @@ -812,8 +812,8 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { validExtensions.add(".gitignore"); // Special case, since subdirs in schemas/ can have any name - if (isSubDirInSchemas(relativeDirectory)) - validExtensions.add(".profile"); + if (isSchemasSubDir(relativeDirectory)) + validExtensions.add(RANKPROFILE_NAME_SUFFIX); else validExtensions.addAll(validFileExtensions.entrySet().stream() .filter(entry -> entry.getKey() @@ -824,13 +824,13 @@ public class FilesApplicationPackage extends AbstractApplicationPackage { return validExtensions; } - private boolean isSubDirInSchemas(java.nio.file.Path relativeDirectory) { + private boolean isSchemasSubDir(java.nio.file.Path relativeDirectory) { java.nio.file.Path schemasPath = SCHEMAS_DIR.toFile().toPath().getName(0); java.nio.file.Path searchDefinitionsPath = SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0); - if (relativeDirectory.equals(schemasPath) ||relativeDirectory.equals(searchDefinitionsPath)) return false; + if (List.of(schemasPath, searchDefinitionsPath).contains(relativeDirectory)) return false; return (relativeDirectory.startsWith(schemasPath + "/") - || relativeDirectory.startsWith(SEARCH_DEFINITIONS_DIR.toFile().toPath().getName(0) + "/")); + || relativeDirectory.startsWith(searchDefinitionsPath + "/")); } } diff --git a/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile b/config-application-package/src/test/resources/app-with-deployment/schemas/default.profile index a0ab4d8225e..a0ab4d8225e 100644 --- a/config-application-package/src/test/resources/app-with-deployment/schemas/music/default.profile +++ b/config-application-package/src/test/resources/app-with-deployment/schemas/default.profile diff --git a/config-application-package/src/test/resources/app-with-deployment/schemas/music/in-subdir.profile b/config-application-package/src/test/resources/app-with-deployment/schemas/music/in-subdir.profile new file mode 100644 index 00000000000..be6a4449132 --- /dev/null +++ b/config-application-package/src/test/resources/app-with-deployment/schemas/music/in-subdir.profile @@ -0,0 +1,5 @@ +rank-profile in-subdir { + second-phase { + expression: fieldMatch(f) + } +} diff --git a/config-bundle/CMakeLists.txt b/config-bundle/CMakeLists.txt index 774372de9d0..9647f65cfce 100644 --- a/config-bundle/CMakeLists.txt +++ b/config-bundle/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(config-bundle) +install_jar(config-bundle-jar-with-dependencies.jar) diff --git a/config-model-api/CMakeLists.txt b/config-model-api/CMakeLists.txt index 81bf9928a8c..a3d3a80ed3f 100644 --- a/config-model-api/CMakeLists.txt +++ b/config-model-api/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(config-model-api) +install_jar(config-model-api-jar-with-dependencies.jar) diff --git a/config-model-api/abi-spec.json b/config-model-api/abi-spec.json index 6e02b88e22e..13988d81e86 100644 --- a/config-model-api/abi-spec.json +++ b/config-model-api/abi-spec.json @@ -156,6 +156,7 @@ "public static final com.yahoo.path.Path SECURITY_DIR", "public static final java.lang.String SD_NAME_SUFFIX", "public static final java.lang.String RANKEXPRESSION_NAME_SUFFIX", + "public static final java.lang.String RANKPROFILE_NAME_SUFFIX", "public static final java.lang.String RULES_NAME_SUFFIX", "public static final java.lang.String EXT_DIR", "public static final java.lang.String PERMANENT_SERVICES" @@ -195,7 +196,7 @@ "public" ], "methods": [ - "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$RevisionTarget, com.yahoo.config.application.api.DeploymentSpec$RevisionChange, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, int, int, int, java.util.List, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", + "public void <init>(com.yahoo.config.provision.InstanceName, java.util.List, com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy, com.yahoo.config.application.api.DeploymentSpec$RevisionTarget, com.yahoo.config.application.api.DeploymentSpec$RevisionChange, com.yahoo.config.application.api.DeploymentSpec$UpgradeRollout, int, int, int, java.util.List, java.util.Optional, java.util.Optional, java.util.Optional, com.yahoo.config.application.api.Notifications, java.util.List, java.time.Instant)", "public com.yahoo.config.provision.InstanceName name()", "public com.yahoo.config.application.api.DeploymentSpec$UpgradePolicy upgradePolicy()", "public com.yahoo.config.application.api.DeploymentSpec$RevisionTarget revisionTarget()", @@ -209,6 +210,7 @@ "public boolean canUpgradeAt(java.time.Instant)", "public boolean canChangeRevisionAt(java.time.Instant)", "public java.util.Optional athenzService(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", + "public java.util.Optional cloudAccount(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", "public com.yahoo.config.application.api.Notifications notifications()", "public java.util.List endpoints()", "public boolean deploysTo(com.yahoo.config.provision.Environment, com.yahoo.config.provision.RegionName)", @@ -258,12 +260,13 @@ ], "methods": [ "public void <init>(com.yahoo.config.provision.Environment)", - "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, boolean, java.util.Optional, java.util.Optional)", + "public void <init>(com.yahoo.config.provision.Environment, java.util.Optional, boolean, java.util.Optional, java.util.Optional, java.util.Optional)", "public com.yahoo.config.provision.Environment environment()", "public java.util.Optional region()", "public boolean active()", "public java.util.Optional testerFlavor()", "public java.util.Optional athenzService()", + "public java.util.Optional cloudAccount()", "public java.util.List zones()", "public boolean concerns(com.yahoo.config.provision.Environment, java.util.Optional)", "public boolean isTest()", diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java index 18d40025af7..0600a31e9c5 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/ApplicationPackage.java @@ -67,6 +67,7 @@ public interface ApplicationPackage { String SD_NAME_SUFFIX = ".sd"; String RANKEXPRESSION_NAME_SUFFIX = ".expression"; + String RANKPROFILE_NAME_SUFFIX = ".profile"; String RULES_NAME_SUFFIX = ".sr"; String EXT_DIR = "ext"; diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java index 9d90167a0ef..8cb70d50a59 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java @@ -1,9 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.application.api; -import ai.vespa.validation.Validation; -import com.yahoo.config.application.api.DeploymentSpec.RevisionTarget; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -14,11 +13,9 @@ import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -29,7 +26,6 @@ import static com.yahoo.config.application.api.DeploymentSpec.RevisionChange.whe import static com.yahoo.config.application.api.DeploymentSpec.RevisionTarget.next; import static com.yahoo.config.provision.Environment.prod; import static java.util.stream.Collectors.toList; -import static java.util.stream.Collectors.toSet; /** * The deployment spec for an application instance @@ -54,6 +50,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { private final List<DeploymentSpec.ChangeBlocker> changeBlockers; private final Optional<String> globalServiceId; private final Optional<AthenzService> athenzService; + private final Optional<CloudAccount> cloudAccount; private final Notifications notifications; private final List<Endpoint> endpoints; @@ -67,25 +64,29 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { List<DeploymentSpec.ChangeBlocker> changeBlockers, Optional<String> globalServiceId, Optional<AthenzService> athenzService, + Optional<CloudAccount> cloudAccount, Notifications notifications, List<Endpoint> endpoints, Instant now) { super(steps); - this.name = name; - this.upgradePolicy = upgradePolicy; + this.name = Objects.requireNonNull(name); + this.upgradePolicy = Objects.requireNonNull(upgradePolicy); + Objects.requireNonNull(revisionTarget); + Objects.requireNonNull(revisionChange); this.revisionTarget = require(maxRisk == 0 || revisionTarget == next, revisionTarget, "revision-target must be 'next' when max-risk is specified"); this.revisionChange = require(maxRisk == 0 || revisionChange == whenClear, revisionChange, "revision-change must be 'when-clear' when max-risk is specified"); - this.upgradeRollout = upgradeRollout; + this.upgradeRollout = Objects.requireNonNull(upgradeRollout); this.minRisk = requireAtLeast(minRisk, "minimum risk score", 0); this.maxRisk = require(maxRisk >= minRisk, maxRisk, "maximum risk cannot be less than minimum risk score"); this.maxIdleHours = requireInRange(maxIdleHours, "maximum idle hours", 0, 168); - this.changeBlockers = changeBlockers; - this.globalServiceId = globalServiceId; - this.athenzService = athenzService; - this.notifications = notifications; - this.endpoints = List.copyOf(endpoints); + this.changeBlockers = Objects.requireNonNull(changeBlockers); + this.globalServiceId = Objects.requireNonNull(globalServiceId); + this.athenzService = Objects.requireNonNull(athenzService); + this.cloudAccount = Objects.requireNonNull(cloudAccount); + this.notifications = Objects.requireNonNull(notifications); + this.endpoints = List.copyOf(Objects.requireNonNull(endpoints)); validateZones(new HashSet<>(), new HashSet<>(), this); validateEndpoints(steps(), globalServiceId, this.endpoints); validateChangeBlockers(changeBlockers, now); @@ -224,6 +225,16 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps { .or(() -> this.athenzService); } + /** Returns the cloud account to use for given environment and region, if any */ + public Optional<CloudAccount> cloudAccount(Environment environment, RegionName region) { + if (!environment.isProduction()) return Optional.empty(); + return zones().stream() + .filter(zone -> zone.concerns(environment, Optional.of(region))) + .findFirst() + .flatMap(DeploymentSpec.DeclaredZone::cloudAccount) + .or(() -> cloudAccount); + } + /** Returns the notification configuration of these instances */ public Notifications notifications() { return notifications; } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index e5ea65b6d4e..22ffdeb7262 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -5,6 +5,7 @@ import com.yahoo.collections.Comparables; import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -362,22 +363,27 @@ public class DeploymentSpec { private final boolean active; private final Optional<AthenzService> athenzService; private final Optional<String> testerFlavor; + private final Optional<CloudAccount> cloudAccount; public DeclaredZone(Environment environment) { - this(environment, Optional.empty(), false, Optional.empty(), Optional.empty()); + this(environment, Optional.empty(), false, Optional.empty(), Optional.empty(), Optional.empty()); } public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active, - Optional<AthenzService> athenzService, Optional<String> testerFlavor) { + Optional<AthenzService> athenzService, Optional<String> testerFlavor, + Optional<CloudAccount> cloudAccount) { if (environment != Environment.prod && region.isPresent()) illegal("Non-prod environments cannot specify a region"); if (environment == Environment.prod && region.isEmpty()) illegal("Prod environments must be specified with a region"); - this.environment = environment; - this.region = region; + if (environment != Environment.prod && cloudAccount.isPresent()) + illegal("Non-prod environments cannot specify cloud account"); + this.environment = Objects.requireNonNull(environment); + this.region = Objects.requireNonNull(region); this.active = active; - this.athenzService = athenzService; - this.testerFlavor = testerFlavor; + this.athenzService = Objects.requireNonNull(athenzService); + this.testerFlavor = Objects.requireNonNull(testerFlavor); + this.cloudAccount = Objects.requireNonNull(cloudAccount); } public Environment environment() { return environment; } @@ -392,6 +398,10 @@ public class DeploymentSpec { public Optional<AthenzService> athenzService() { return athenzService; } + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + @Override public List<DeclaredZone> zones() { return Collections.singletonList(this); } diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java index 96cc33d44b4..2f73bff83b8 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java @@ -21,6 +21,7 @@ import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.application.api.TimeWindow; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -72,6 +73,8 @@ public class DeploymentSpecXmlReader { private static final String athenzDomainAttribute = "athenz-domain"; private static final String testerFlavorAttribute = "tester-flavor"; private static final String majorVersionAttribute = "major-version"; + private static final String globalServiceIdAttribute = "global-service-id"; + private static final String cloudAccountAttribute = "cloud-account"; private final boolean validate; private final Clock clock; @@ -118,7 +121,7 @@ public class DeploymentSpecXmlReader { List<Step> steps = new ArrayList<>(); List<Endpoint> applicationEndpoints = List.of(); if ( ! containsTag(instanceTag, root)) { // deployment spec skipping explicit instance -> "default" instance - steps.addAll(readInstanceContent("default", root, new MutableOptional<>(), root)); + steps.addAll(readInstanceContent("default", root, new HashMap<>(), root)); } else { if (XML.getChildren(root).stream().anyMatch(child -> child.getTagName().equals(prodTag))) @@ -129,9 +132,9 @@ public class DeploymentSpecXmlReader { for (Element child : XML.getChildren(root)) { String tagName = child.getTagName(); if (tagName.equals(instanceTag)) { - steps.addAll(readInstanceContent(child.getAttribute(idAttribute), child, new MutableOptional<>(), root)); + steps.addAll(readInstanceContent(child.getAttribute(idAttribute), child, new HashMap<>(), root)); } else { - steps.addAll(readNonInstanceSteps(child, new MutableOptional<>(), root)); // (No global service id here) + steps.addAll(readNonInstanceSteps(child, new HashMap<>(), root)); // (No global service id here) } } applicationEndpoints = readEndpoints(root, Optional.empty(), steps); @@ -156,7 +159,7 @@ public class DeploymentSpecXmlReader { */ private List<DeploymentInstanceSpec> readInstanceContent(String instanceNameString, Element instanceTag, - MutableOptional<String> globalServiceId, + Map<String, String> prodAttributes, Element parentTag) { if (instanceNameString.isBlank()) illegal("<instance> attribute 'id' must be specified, and not be blank"); @@ -178,12 +181,13 @@ public class DeploymentSpecXmlReader { int maxIdleHours = getWithFallback(instanceTag, parentTag, upgradeTag, "max-idle-hours", Integer::parseInt, 8); List<DeploymentSpec.ChangeBlocker> changeBlockers = readChangeBlockers(instanceTag, parentTag); Optional<AthenzService> athenzService = mostSpecificAttribute(instanceTag, athenzServiceAttribute).map(AthenzService::from); + Optional<CloudAccount> cloudAccount = mostSpecificAttribute(instanceTag, cloudAccountAttribute).map(CloudAccount::new); Notifications notifications = readNotifications(instanceTag, parentTag); // Values where there is no default List<Step> steps = new ArrayList<>(); for (Element instanceChild : XML.getChildren(instanceTag)) - steps.addAll(readNonInstanceSteps(instanceChild, globalServiceId, instanceChild)); + steps.addAll(readNonInstanceSteps(instanceChild, prodAttributes, instanceChild)); List<Endpoint> endpoints = readEndpoints(instanceTag, Optional.of(instanceNameString), steps); // Build and return instances with these values @@ -198,43 +202,46 @@ public class DeploymentSpecXmlReader { upgradeRollout, minRisk, maxRisk, maxIdleHours, changeBlockers, - globalServiceId.asOptional(), + Optional.ofNullable(prodAttributes.get(globalServiceIdAttribute)), athenzService, + cloudAccount, notifications, endpoints, now)) .collect(Collectors.toList()); } - private List<Step> readSteps(Element stepTag, MutableOptional<String> globalServiceId, Element parentTag) { + private List<Step> readSteps(Element stepTag, Map<String, String> prodAttributes, Element parentTag) { if (stepTag.getTagName().equals(instanceTag)) - return new ArrayList<>(readInstanceContent(stepTag.getAttribute(idAttribute), stepTag, globalServiceId, parentTag)); + return new ArrayList<>(readInstanceContent(stepTag.getAttribute(idAttribute), stepTag, prodAttributes, parentTag)); else - return readNonInstanceSteps(stepTag, globalServiceId, parentTag); + return readNonInstanceSteps(stepTag, prodAttributes, parentTag); } // Consume the given tag as 0-N steps. 0 if it is not a step, >1 if it contains multiple nested steps that should be flattened - @SuppressWarnings("fallthrough") - private List<Step> readNonInstanceSteps(Element stepTag, MutableOptional<String> globalServiceId, Element parentTag) { + private List<Step> readNonInstanceSteps(Element stepTag, Map<String, String> prodAttributes, Element parentTag) { Optional<AthenzService> athenzService = mostSpecificAttribute(stepTag, athenzServiceAttribute).map(AthenzService::from); Optional<String> testerFlavor = mostSpecificAttribute(stepTag, testerFlavorAttribute); - if (prodTag.equals(stepTag.getTagName())) - globalServiceId.set(readGlobalServiceId(stepTag)); - else if (readGlobalServiceId(stepTag).isPresent()) - illegal("Attribute 'global-service-id' is only valid on 'prod' tag."); + if (prodTag.equals(stepTag.getTagName())) { + readGlobalServiceId(stepTag).ifPresent(id -> prodAttributes.put(globalServiceIdAttribute, id)); + } else { + if (readGlobalServiceId(stepTag).isPresent()) illegal("Attribute '" + globalServiceIdAttribute + "' is only valid on 'prod' tag"); + } switch (stepTag.getTagName()) { case testTag: if (Stream.iterate(stepTag, Objects::nonNull, Node::getParentNode) - .anyMatch(node -> prodTag.equals(node.getNodeName()))) + .anyMatch(node -> prodTag.equals(node.getNodeName()))) { return List.of(new DeclaredTest(RegionName.from(XML.getValue(stepTag).trim()))); + } + return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, Optional.empty())); case stagingTag: - return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor)); + return List.of(new DeclaredZone(Environment.from(stepTag.getTagName()), Optional.empty(), false, athenzService, testerFlavor, Optional.empty())); case prodTag: // regions, delay and parallel may be nested within, but we can flatten them return XML.getChildren(stepTag).stream() - .flatMap(child -> readNonInstanceSteps(child, globalServiceId, stepTag).stream()) + .flatMap(child -> readNonInstanceSteps(child, prodAttributes, stepTag).stream()) .collect(Collectors.toList()); case delayTag: return List.of(new Delay(Duration.ofSeconds(longAttribute("hours", stepTag) * 60 * 60 + @@ -242,11 +249,11 @@ public class DeploymentSpecXmlReader { longAttribute("seconds", stepTag)))); case parallelTag: // regions and instances may be nested within return List.of(new ParallelSteps(XML.getChildren(stepTag).stream() - .flatMap(child -> readSteps(child, globalServiceId, parentTag).stream()) + .flatMap(child -> readSteps(child, prodAttributes, parentTag).stream()) .collect(Collectors.toList()))); case stepsTag: // regions and instances may be nested within return List.of(new Steps(XML.getChildren(stepTag).stream() - .flatMap(child -> readSteps(child, globalServiceId, parentTag).stream()) + .flatMap(child -> readSteps(child, prodAttributes, parentTag).stream()) .collect(Collectors.toList()))); case regionTag: return List.of(readDeclaredZone(Environment.prod, athenzService, testerFlavor, stepTag)); @@ -425,13 +432,14 @@ public class DeploymentSpecXmlReader { private DeclaredZone readDeclaredZone(Environment environment, Optional<AthenzService> athenzService, Optional<String> testerFlavor, Element regionTag) { return new DeclaredZone(environment, Optional.of(RegionName.from(XML.getValue(regionTag).trim())), - readActive(regionTag), athenzService, testerFlavor); + readActive(regionTag), athenzService, testerFlavor, + stringAttribute(cloudAccountAttribute, regionTag).map(CloudAccount::new)); } private Optional<String> readGlobalServiceId(Element environmentTag) { - String globalServiceId = environmentTag.getAttribute("global-service-id"); + String globalServiceId = environmentTag.getAttribute(globalServiceIdAttribute); if (globalServiceId.isEmpty()) return Optional.empty(); - deprecate(environmentTag, List.of("global-service-id"), "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); + deprecate(environmentTag, List.of(globalServiceIdAttribute), "See https://cloud.vespa.ai/en/reference/routing#deprecated-syntax"); return Optional.of(globalServiceId); } @@ -546,14 +554,4 @@ public class DeploymentSpecXmlReader { throw new IllegalArgumentException(message); } - private static class MutableOptional<T> { - - private Optional<T> value = Optional.empty(); - - public void set(Optional<T> value) { this.value = value; } - - public Optional<T> asOptional() { return value; } - - } - } diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 855e445de22..3747115ae96 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.HostName; @@ -86,6 +87,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int maxUnCommittedMemory() { return 130000; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { return 16; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { return 100; } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean loadCodeAsHugePages() { return false; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default List<String> allowedAthenzProxyIdentities() { return List.of(); } @@ -99,7 +101,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } - @ModelFeatureFlag(owners = {"hmusum"}) default boolean failDeploymentWithInvalidJvmOptions() { return false; } + @ModelFeatureFlag(owners = {"hmusum"}, removeAfter = "7.587") default boolean failDeploymentWithInvalidJvmOptions() { return true; } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @@ -157,6 +159,9 @@ public interface ModelContext { default List<String> zoneDnsSuffixes() { return List.of(); } List<String> environmentVariables(); + + default Optional<CloudAccount> cloudAccount() { return Optional.empty(); } + } @Retention(RetentionPolicy.RUNTIME) diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 5073c6b9fb2..7f0e9b4cae8 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -3,6 +3,7 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableSet; import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -1509,6 +1510,29 @@ public class DeploymentSpecTest { "</deployment>").deployableHashCode()); } + @Test + public void cloudAccount() { + StringReader r = new StringReader( + "<deployment version='1.0' cloud-account='100000000000'>" + + " <instance id='beta' cloud-account='200000000000'>" + + " <prod>" + + " <region>us-west-1</region>" + + " </prod>" + + " </instance>" + + " <instance id='main'>" + + " <prod>" + + " <region cloud-account='300000000000'>us-east-1</region>" + + " <region>eu-west-1</region>" + + " </prod>" + + " </instance>" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + assertEquals(Optional.of(new CloudAccount("200000000000")), spec.requireInstance("beta").cloudAccount(Environment.prod, RegionName.from("us-west-1"))); + assertEquals(Optional.of(new CloudAccount("300000000000")), spec.requireInstance("main").cloudAccount(Environment.prod, RegionName.from("us-east-1"))); + assertEquals(Optional.of(new CloudAccount("100000000000")), spec.requireInstance("main").cloudAccount(Environment.prod, RegionName.from("eu-west-1"))); + } + private static void assertInvalid(String deploymentSpec, String errorMessagePart) { assertInvalid(deploymentSpec, errorMessagePart, new ManualClock()); } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java index efed8ecc06c..1232f700fee 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java @@ -2,6 +2,7 @@ package com.yahoo.config.application.api; import com.google.common.collect.ImmutableSet; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import org.junit.Test; @@ -702,6 +703,35 @@ public class DeploymentSpecWithoutInstanceTest { assertEquals(Set.of("us-east", "us-west"), endpointRegions("default", spec)); } + @Test + public void productionSpecWithCloudAccount() { + StringReader r = new StringReader( + "<deployment version='1.0' cloud-account='012345678912'>" + + " <prod>" + + " <region cloud-account='219876543210'>us-east-1</region>" + + " <region>us-west-1</region>" + + " </prod>" + + "</deployment>" + ); + DeploymentSpec spec = DeploymentSpec.fromXml(r); + DeploymentInstanceSpec instance = spec.requireInstance("default"); + assertEquals(Optional.of(new CloudAccount("219876543210")), instance.cloudAccount(Environment.prod, RegionName.from("us-east-1"))); + assertEquals(Optional.of(new CloudAccount("012345678912")), instance.cloudAccount(Environment.prod, RegionName.from("us-west-1"))); + assertEquals(Optional.empty(), instance.cloudAccount(Environment.staging, RegionName.defaultName())); + + r = new StringReader( + "<deployment version='1.0'>" + + " <prod>" + + " <region cloud-account='219876543210'>us-east-1</region>" + + " <region>us-west-1</region>" + + " </prod>" + + "</deployment>" + ); + spec = DeploymentSpec.fromXml(r); + assertEquals(Optional.of(new CloudAccount("219876543210")), spec.requireInstance("default").cloudAccount(Environment.prod, RegionName.from("us-east-1"))); + assertEquals(Optional.empty(), spec.requireInstance("default").cloudAccount(Environment.prod, RegionName.from("us-west-1"))); + } + private static Set<String> endpointRegions(String endpointId, DeploymentSpec spec) { return spec.requireInstance("default").endpoints().stream() .filter(endpoint -> endpoint.endpointId().equals(endpointId)) diff --git a/config-model-fat/CMakeLists.txt b/config-model-fat/CMakeLists.txt index fd40048e2c4..a2f59cd51c8 100644 --- a/config-model-fat/CMakeLists.txt +++ b/config-model-fat/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact(config-model-fat) +install_jar(config-model-fat.jar) install(FILES src/main/resources/config-models.xml DESTINATION conf/configserver-app) diff --git a/config-model-fat/pom.xml b/config-model-fat/pom.xml index 5f36a26ccef..96cafd4ec0f 100644 --- a/config-model-fat/pom.xml +++ b/config-model-fat/pom.xml @@ -72,7 +72,7 @@ <_fixupmessages>"Classes found in the wrong directory"</_fixupmessages> <!-- Hide warnings for multi-release jars --> <Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName> <Bundle-Version>${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}</Bundle-Version> - <Embed-Dependency>*;scope=compile|runtime</Embed-Dependency> + <Embed-Dependency>*;scope=compile|runtime;type=!pom</Embed-Dependency> <Embed-Transitive>true</Embed-Transitive> <Import-Package>!*</Import-Package> <!-- Should only import packages that meet one or more of the below criteria: diff --git a/config-model/CMakeLists.txt b/config-model/CMakeLists.txt index d23ea29c3d0..8f59bb77ef2 100644 --- a/config-model/CMakeLists.txt +++ b/config-model/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(config-model) +install_jar(config-model-jar-with-dependencies.jar) vespa_install_script(src/main/perl/vespa-deploy bin) vespa_install_script(src/main/perl/vespa-expand-config.pl bin) diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java index 9eef1dc2447..11d84884194 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/DeployState.java @@ -28,10 +28,11 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.Application; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionBuilder; import com.yahoo.vespa.config.ConfigDefinitionKey; @@ -216,7 +217,7 @@ public class DeployState implements ConfigDefinitionStore { for (var entry : importedModels.getSkippedModels().entrySet()) { // TODO: Vespa 8: Throw IllegalArgumentException instead deployLogger.logApplicationPackage(Level.WARNING, "Skipping import of model " + entry.getKey() + " as an exception " + - "occurred during import. Error: " + entry.getValue()); + "occurred during import: " + entry.getValue()); } return importedModels; } @@ -334,6 +335,8 @@ public class DeployState implements ConfigDefinitionStore { private boolean accessLoggingEnabledByDefault = true; private Optional<DockerImage> wantedDockerImageRepo = Optional.empty(); private Reindexing reindexing = null; + private RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); + private QueryProfiles queryProfiles = null; public Builder() {} @@ -431,6 +434,21 @@ public class DeployState implements ConfigDefinitionStore { return this; } + public Builder rankProfileRegistry(RankProfileRegistry rankProfileRegistry) { + this.rankProfileRegistry = rankProfileRegistry; + return this; + } + + public Builder queryProfiles(QueryProfiles queryProfiles) { + this.queryProfiles = queryProfiles; + return this; + } + + public Builder queryProfiles(QueryProfileRegistry queryProfileRegistry) { + this.queryProfiles = new QueryProfiles(queryProfileRegistry, logger); + return this; + } + public Builder reindexing(Reindexing reindexing) { this.reindexing = Objects.requireNonNull(reindexing); return this; } public DeployState build() { @@ -438,8 +456,8 @@ public class DeployState implements ConfigDefinitionStore { } public DeployState build(ValidationParameters validationParameters) { - RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - QueryProfiles queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); + if (queryProfiles == null) + queryProfiles = new QueryProfilesBuilder().build(applicationPackage, logger); SemanticRules semanticRules = new SemanticRuleBuilder().build(applicationPackage); Application application = new ApplicationBuilder(applicationPackage, fileRegistry, logger, properties, rankProfileRegistry, queryProfiles.getRegistry()) diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 37f3a2ef043..aa8a0f6e1ec 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -66,7 +66,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean unorderedMergeChaining = true; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; - private boolean failDeploymentWithInvalidJvmOptions = false; private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; @@ -122,7 +121,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } - @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } @@ -299,11 +297,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties failDeploymentWithInvalidJvmOptions(boolean fail) { - failDeploymentWithInvalidJvmOptions = fail; - return this; - } - public TestProperties setMergeThrottlingPolicy(String policy) { this.mergeThrottlingPolicy = policy; return this; diff --git a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java index 1838f1e36b7..8892de0a88d 100644 --- a/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java +++ b/config-model/src/main/java/com/yahoo/config/model/test/MockApplicationPackage.java @@ -6,9 +6,6 @@ import com.yahoo.config.application.api.ComponentInfo; import com.yahoo.config.application.api.UnparsedConfigDefinition; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.component.Version; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.InstanceName; @@ -18,9 +15,6 @@ import com.yahoo.path.Path; import com.yahoo.io.reader.NamedReader; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.config.application.api.ApplicationPackage; @@ -37,7 +31,9 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -53,13 +49,14 @@ public class MockApplicationPackage implements ApplicationPackage { public static final String DEPLOYED_BY_USER = "user"; public static final String APPLICATION_NAME = "application"; public static final long APPLICATION_GENERATION = 1L; - public static final String MUSIC_SEARCHDEFINITION = createSearchDefinition("music", "foo"); - public static final String BOOK_SEARCHDEFINITION = createSearchDefinition("book", "bar"); + public static final String MUSIC_SCHEMA = createSchema("music", "foo"); + public static final String BOOK_SCHEMA = createSchema("book", "bar"); private final File root; private final String hostsS; private final String servicesS; private final List<String> schemas; + private final Map<Path, MockApplicationFile> files; private final String schemaDir; private final Optional<String> deploymentSpec; private final Optional<String> validationOverrides; @@ -68,6 +65,7 @@ public class MockApplicationPackage implements ApplicationPackage { private final ApplicationMetaData applicationMetaData; protected MockApplicationPackage(File root, String hosts, String services, List<String> schemas, + Map<Path, MockApplicationFile> files, String schemaDir, String deploymentSpec, String validationOverrides, boolean failOnValidateXml, String queryProfile, String queryProfileType) { @@ -75,6 +73,7 @@ public class MockApplicationPackage implements ApplicationPackage { this.hostsS = hosts; this.servicesS = services; this.schemas = schemas; + this.files = files; this.schemaDir = schemaDir; this.deploymentSpec = Optional.ofNullable(deploymentSpec); this.validationOverrides = Optional.ofNullable(validationOverrides); @@ -135,7 +134,7 @@ public class MockApplicationPackage implements ApplicationPackage { throw new IllegalArgumentException("Expected the first line of a schema but got '" + sd + "'"); int end = s.indexOf(' '); if (end < 0) - end = s.indexOf('}'); + end = s.indexOf('}'); return s.substring(0, end).trim(); } @@ -146,15 +145,21 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<NamedReader> getFiles(Path dir, String fileSuffix, boolean recurse) { + if (dir.elements().contains(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getName())) + return List.of(); // No legacy paths + return getFiles(new File(root, dir.getName()), fileSuffix, recurse); + } + + private List<NamedReader> getFiles(File dir, String fileSuffix, boolean recurse) { try { - if (dir.elements().contains(ApplicationPackage.SEARCH_DEFINITIONS_DIR.getName())) return List.of(); // No legacy paths - File dirFile = new File(root, dir.getName()); - if ( ! dirFile.exists()) return List.of(); - if (recurse) throw new RuntimeException("Recurse not implemented"); + if ( ! dir.exists()) return List.of(); List<NamedReader> readers = new ArrayList<>(); - for (var i = Files.list(dirFile.toPath()).filter(p -> p.getFileName().toString().endsWith(fileSuffix)).iterator(); i.hasNext(); ) { + for (var i = Files.list(dir.toPath()).iterator(); i.hasNext(); ) { var file = i.next(); - readers.add(new NamedReader(file.toString(), IOUtils.createReader(file.toString()))); + if (file.getFileName().toString().endsWith(fileSuffix)) + readers.add(new NamedReader(file.toString(), IOUtils.createReader(file.toString()))); + else if (recurse) + readers.addAll(getFiles(new File(dir, file.getFileName().toString()), fileSuffix, recurse)); } return readers; } @@ -165,6 +170,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile getFile(Path file) { + if (files.containsKey(file)) return files.get(file); return new MockApplicationFile(file, Path.fromString(root.toString())); } @@ -239,6 +245,7 @@ public class MockApplicationPackage implements ApplicationPackage { private String hosts = null; private String services = null; private List<String> schemas = Collections.emptyList(); + private Map<Path, MockApplicationFile> files = new LinkedHashMap<>(); private String schemaDir = null; private String deploymentSpec = null; private String validationOverrides = null; @@ -282,6 +289,16 @@ public class MockApplicationPackage implements ApplicationPackage { return this; } + /** Additional (mock) files that will exist in this application package, with their content. */ + public Builder withFiles(Map<Path, String> files) { + Map<Path, MockApplicationFile> mockFiles = new HashMap<>(); + for (var file : files.entrySet()) + mockFiles.put(file.getKey(), new MockApplicationFile(file.getKey(), + Path.fromString(root.toString()), file.getValue())); + this.files = mockFiles; + return this; + } + public Builder withSchemaDir(String schemaDir) { this.schemaDir = schemaDir; return this; @@ -313,13 +330,13 @@ public class MockApplicationPackage implements ApplicationPackage { } public ApplicationPackage build() { - return new MockApplicationPackage(root, hosts, services, schemas, schemaDir, - deploymentSpec, validationOverrides, failOnValidateXml, - queryProfile, queryProfileType); + return new MockApplicationPackage(root, hosts, services, schemas, files, schemaDir, + deploymentSpec, validationOverrides, failOnValidateXml, + queryProfile, queryProfileType); } } - public static String createSearchDefinition(String name, String fieldName) { + public static String createSchema(String name, String fieldName) { return "search " + name + " {" + " document " + name + " {" + " field " + fieldName + " type string {}" + @@ -369,25 +386,36 @@ public class MockApplicationPackage implements ApplicationPackage { /** The File pointing to the actual file represented by this */ private final File file; + /** The content of this file, or null to read it from the file system. */ + private final String content; + public MockApplicationFile(Path filePath, Path applicationPackagePath) { + this(filePath, applicationPackagePath, null); + } + + private MockApplicationFile(Path filePath, Path applicationPackagePath, String content) { super(filePath); this.root = applicationPackagePath; file = applicationPackagePath.append(filePath).toFile(); + this.content = content; } @Override public boolean isDirectory() { + if (content != null) return false; return file.isDirectory(); } @Override public boolean exists() { + if (content != null) return true; return file.exists(); } @Override public Reader createReader() { try { + if (content != null) return new StringReader(content); if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); return IOUtils.createReader(file, "UTF-8"); } @@ -399,6 +427,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public InputStream createInputStream() { try { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); if ( ! exists()) throw new FileNotFoundException("File '" + file + "' does not exist"); return new BufferedInputStream(new FileInputStream(file)); } @@ -416,6 +445,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile writeFile(Reader input) { try { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); IOUtils.writeFile(file, IOUtils.readAll(input), false); return this; } @@ -427,6 +457,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile appendFile(String value) { try { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); IOUtils.writeFile(file, value, true); return this; } @@ -437,7 +468,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public List<ApplicationFile> listFiles(PathFilter filter) { - if ( ! isDirectory()) return Collections.emptyList(); + if ( ! isDirectory()) return List.of(); return Arrays.stream(file.listFiles()).filter(f -> filter.accept(Path.fromString(f.toString()))) .map(f -> new MockApplicationFile(asApplicationRelativePath(f), root)) .collect(Collectors.toList()); @@ -445,6 +476,7 @@ public class MockApplicationPackage implements ApplicationPackage { @Override public ApplicationFile delete() { + if (content != null) throw new UnsupportedOperationException("Not implemented for mock file content"); file.delete(); return this; } diff --git a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java index ff5930f8f6a..7f15da3179f 100644 --- a/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java +++ b/config-model/src/main/java/com/yahoo/documentmodel/NewDocumentType.java @@ -4,16 +4,15 @@ package com.yahoo.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Document; import com.yahoo.document.Field; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.StructuredDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.document.datatypes.FieldValue; -import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.processing.BuiltInFieldSets; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.processing.BuiltInFieldSets; import java.util.Collection; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java b/config-model/src/main/java/com/yahoo/schema/Application.java index 2dda670f07c..aa47818ff99 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Application.java +++ b/config-model/src/main/java/com/yahoo/schema/Application.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.derived.SearchOrderer; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.processing.Processing; -import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.schema.derived.SearchOrderer; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.processing.Processing; +import com.yahoo.schema.processing.Processor; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java index 51c183a21d9..d2c7035bac6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ApplicationBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/ApplicationBuilder.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -14,10 +14,10 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; -import com.yahoo.searchdefinition.parser.ConvertSchemaCollection; -import com.yahoo.searchdefinition.parser.IntermediateCollection; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.schema.parser.ConvertSchemaCollection; +import com.yahoo.schema.parser.IntermediateCollection; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.Processor; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.model.container.search.QueryProfiles; @@ -197,7 +197,8 @@ public class ApplicationBuilder { } Path rankProfilePath = ApplicationPackage.SCHEMAS_DIR.append(schemaName); - for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile")) { + for (NamedReader reader : applicationPackage.getFiles(rankProfilePath, ".profile", true)) { + System.out.println("Got " + reader); mediator.addRankProfileFile(schemaName, reader); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java b/config-model/src/main/java/com/yahoo/schema/DefaultRankProfile.java index 812726a609e..9ab03b8c4a1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DefaultRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/DefaultRankProfile.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.document.ImmutableSDField; import java.util.LinkedHashSet; -import java.util.List; import java.util.Set; /** @@ -18,11 +17,11 @@ public class DefaultRankProfile extends RankProfile { /** * Creates a new rank profile * - * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param rankProfileRegistry the {@link com.yahoo.schema.RankProfileRegistry} * to use for storing and looking up rank profiles */ - public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { - super("default", schema, rankProfileRegistry, rankingConstants); + public DefaultRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { + super("default", schema, rankProfileRegistry); } /** Ignore self inheriting of default as some applications may use that for historical reasons. */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java index 11c55521100..7a8a3963ba4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DistributableResource.java +++ b/config-model/src/main/java/com/yahoo/schema/DistributableResource.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.FileReference; import com.yahoo.config.application.api.FileRegistry; @@ -14,7 +14,7 @@ public class DistributableResource implements Comparable <DistributableResource> /** The search definition-unique name of this constant */ private final String name; - //TODO Make path/pathType final + // TODO: Make path/pathType final private PathType pathType; private String path; private FileReference fileReference = new FileReference(""); @@ -35,14 +35,14 @@ public class DistributableResource implements Comparable <DistributableResource> this.pathType = type; } - //TODO Remove and make path/pathType final + // TODO: Remove and make path/pathType final public void setFileName(String fileName) { Objects.requireNonNull(fileName, "Filename cannot be null"); this.path = fileName; this.pathType = PathType.FILE; } - //TODO Remove and make path/pathType final + // TODO: Remove and make path/pathType final public void setUri(String uri) { Objects.requireNonNull(uri, "uri cannot be null"); this.path = uri; @@ -65,7 +65,7 @@ public class DistributableResource implements Comparable <DistributableResource> } } - void register(FileRegistry fileRegistry) { + public void register(FileRegistry fileRegistry) { switch (pathType) { case FILE: fileReference = fileRegistry.addFile(path); @@ -91,4 +91,5 @@ public class DistributableResource implements Comparable <DistributableResource> public int compareTo(DistributableResource o) { return name.compareTo(o.getName()); } + } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentGraphValidator.java b/config-model/src/main/java/com/yahoo/schema/DocumentGraphValidator.java index 33f202a55fa..648cdf18c5b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentGraphValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentGraphValidator.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; import java.util.ArrayDeque; import java.util.Deque; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java b/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java index bfe0db6f65d..1a6134b410f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentModelBuilder.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; @@ -8,7 +8,6 @@ import com.yahoo.document.DocumentType; import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; -import com.yahoo.document.StructuredDataType; import com.yahoo.document.TemporaryStructuredDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; @@ -20,18 +19,17 @@ import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.documentmodel.OwnedTemporaryType; import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedFields; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; -import com.yahoo.searchdefinition.document.annotation.TemporaryAnnotationReferenceDataType; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedFields; +import com.yahoo.schema.document.annotation.SDAnnotationType; +import com.yahoo.schema.document.annotation.TemporaryAnnotationReferenceDataType; import com.yahoo.vespa.documentmodel.DocumentModel; import com.yahoo.vespa.documentmodel.FieldView; import com.yahoo.vespa.documentmodel.SearchDef; import com.yahoo.vespa.documentmodel.SearchField; -import java.util.AbstractMap; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java b/config-model/src/main/java/com/yahoo/schema/DocumentOnlySchema.java index 1d71a9f1494..f9005c7b775 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentOnlySchema.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentOnlySchema.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; /** * A search that was derived from an sd file containing no search element(s), only diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReference.java b/config-model/src/main/java/com/yahoo/schema/DocumentReference.java index 145effcdc9d..048035ffef8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReference.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentReference.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.Field; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java b/config-model/src/main/java/com/yahoo/schema/DocumentReferenceResolver.java index 2e8362f1839..b3e06fd5e02 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferenceResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentReferenceResolver.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.Field; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import java.util.Collection; import java.util.Map; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java b/config-model/src/main/java/com/yahoo/schema/DocumentReferences.java index d0ac967ce7e..3583a5134e0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentReferences.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentReferences.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import java.util.Collections; import java.util.Iterator; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java b/config-model/src/main/java/com/yahoo/schema/DocumentsOnlyRankProfile.java index acab2b96772..ffd517cf241 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/DocumentsOnlyRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/DocumentsOnlyRankProfile.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import java.util.List; @@ -14,9 +14,8 @@ import java.util.List; */ public class DocumentsOnlyRankProfile extends RankProfile { - public DocumentsOnlyRankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, - RankingConstants rankingConstants) { - super(name, schema, rankProfileRegistry, rankingConstants); + public DocumentsOnlyRankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { + super(name, schema, rankProfileRegistry); } @Override diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java b/config-model/src/main/java/com/yahoo/schema/FeatureNames.java index a226fa4812f..0671903194f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FeatureNames.java +++ b/config-model/src/main/java/com/yahoo/schema/FeatureNames.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplier.java index 7fa009bed67..b2f40b045ea 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplier.java +++ b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplier.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForSearch.java index 52110e95ac7..b107dbaea59 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForSearch.java +++ b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForSearch.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForStructs.java index 4a5a858f828..1ec1de6a9c6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldOperationApplierForStructs.java +++ b/config-model/src/main/java/com/yahoo/schema/FieldOperationApplierForStructs.java @@ -1,14 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.document.DataType; -import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; +import com.yahoo.schema.document.SDDocumentType; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java b/config-model/src/main/java/com/yahoo/schema/FieldSets.java index 065ade9b094..0594056150c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/FieldSets.java +++ b/config-model/src/main/java/com/yahoo/schema/FieldSets.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; -import java.util.stream.Collectors; -import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.schema.document.FieldSet; /** * The field sets owned by a {@link Schema} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java b/config-model/src/main/java/com/yahoo/schema/ImmutableSchema.java index b2f46f4a309..c352d9c417f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImmutableSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/ImmutableSchema.java @@ -1,12 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.vespa.documentmodel.SummaryField; import java.io.Reader; @@ -34,9 +35,9 @@ public interface ImmutableSchema { ApplicationPackage applicationPackage(); DeployLogger getDeployLogger(); ModelContext.Properties getDeployProperties(); - RankingConstants rankingConstants(); + Map<Reference, RankProfile.Constant> constants(); LargeRankExpressions rankExpressionFiles(); - OnnxModels onnxModels(); + Map<String, OnnxModel> onnxModels(); Stream<ImmutableSDField> allImportedFields(); SDDocumentType getDocument(); ImmutableSDField getField(String name); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java b/config-model/src/main/java/com/yahoo/schema/ImportedFieldsEnumerator.java index 512cae5021a..0df79b30298 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/ImportedFieldsEnumerator.java +++ b/config-model/src/main/java/com/yahoo/schema/ImportedFieldsEnumerator.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; import java.util.Collection; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java b/config-model/src/main/java/com/yahoo/schema/Index.java index 14f07f224ab..190081cf80f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Index.java +++ b/config-model/src/main/java/com/yahoo/schema/Index.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.Stemming; import java.io.Serializable; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java b/config-model/src/main/java/com/yahoo/schema/LargeRankExpressions.java index 5e3ef39f360..cfdd4729b97 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/LargeRankExpressions.java +++ b/config-model/src/main/java/com/yahoo/schema/LargeRankExpressions.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.FileRegistry; import java.util.Collection; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java b/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java index 23538deed66..c6c807f2dbb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/MapEvaluationTypeContext.java +++ b/config-model/src/main/java/com/yahoo/schema/MapEvaluationTypeContext.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.google.common.collect.ImmutableMap; -import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; -import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer; +import com.yahoo.schema.expressiontransforms.OnnxModelTransformer; +import com.yahoo.schema.expressiontransforms.TokenTransformer; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java index 4b849af9662..26a0b3e595d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModel.java +++ b/config-model/src/main/java/com/yahoo/schema/OnnxModel.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.ml.OnnxModelInfo; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java b/config-model/src/main/java/com/yahoo/schema/RankExpressionBody.java index 6ba17123fb4..d383a25aecb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankExpressionBody.java +++ b/config-model/src/main/java/com/yahoo/schema/RankExpressionBody.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.FileRegistry; import java.nio.ByteBuffer; -import java.util.Objects; import static java.util.Objects.requireNonNull; @@ -26,7 +25,7 @@ public class RankExpressionBody extends DistributableResource { } } - void register(FileRegistry fileRegistry) { + public void register(FileRegistry fileRegistry) { register(fileRegistry, blob); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java b/config-model/src/main/java/com/yahoo/schema/RankProfile.java index 13f9028cc0a..5479ecf323f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfile.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.google.common.collect.ImmutableMap; @@ -10,12 +10,12 @@ import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.expressiontransforms.ExpressionTransforms; -import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.expressiontransforms.ExpressionTransforms; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.parser.ParseException; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -116,22 +116,20 @@ public class RankProfile implements Cloneable { private Map<Reference, Input> inputs = new LinkedHashMap<>(); + private Map<Reference, Constant> constants = new LinkedHashMap<>(); + + private Map<String, OnnxModel> onnxModels = new LinkedHashMap<>(); + private Set<String> filterFields = new HashSet<>(); private final RankProfileRegistry rankProfileRegistry; - /** Constants in ranking expressions */ - private Map<String, Constant> constants = new HashMap<>(); - private final TypeSettings attributeTypes = new TypeSettings(); private List<ImmutableSDField> allFieldsList; private Boolean strict; - /** Global onnx models not tied to a schema */ - private final OnnxModels onnxModels; - private final RankingConstants rankingConstants; private final ApplicationPackage applicationPackage; private final DeployLogger deployLogger; @@ -140,14 +138,12 @@ public class RankProfile implements Cloneable { * * @param name the name of the new profile * @param schema the schema owning this profile - * @param rankProfileRegistry the {@link com.yahoo.searchdefinition.RankProfileRegistry} to use for storing + * @param rankProfileRegistry the {@link com.yahoo.schema.RankProfileRegistry} to use for storing * and looking up rank profiles. */ - public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { + public RankProfile(String name, Schema schema, RankProfileRegistry rankProfileRegistry) { this.name = Objects.requireNonNull(name, "name cannot be null"); this.schema = Objects.requireNonNull(schema, "schema cannot be null"); - this.onnxModels = null; - this.rankingConstants = rankingConstants; this.rankProfileRegistry = rankProfileRegistry; this.applicationPackage = schema.applicationPackage(); this.deployLogger = schema.getDeployLogger(); @@ -159,12 +155,10 @@ public class RankProfile implements Cloneable { * @param name the name of the new profile */ public RankProfile(String name, ApplicationPackage applicationPackage, DeployLogger deployLogger, - RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants, OnnxModels onnxModels) { + RankProfileRegistry rankProfileRegistry) { this.name = Objects.requireNonNull(name, "name cannot be null"); this.schema = null; this.rankProfileRegistry = rankProfileRegistry; - this.rankingConstants = rankingConstants; - this.onnxModels = onnxModels; this.applicationPackage = applicationPackage; this.deployLogger = deployLogger; } @@ -179,15 +173,6 @@ public class RankProfile implements Cloneable { return applicationPackage; } - /** Returns the ranking constants of the owner of this */ - public RankingConstants rankingConstants() { - return rankingConstants; - } - - public Map<String, OnnxModel> onnxModels() { - return schema != null ? schema.onnxModels().asMap() : onnxModels.asMap(); - } - private Stream<ImmutableSDField> allFields() { if (schema == null) return Stream.empty(); if (allFieldsList == null) { @@ -416,28 +401,58 @@ public class RankProfile implements Cloneable { return finalSettings; } - public void addConstant(String name, Constant value) { - constants.put(name, value); + public void add(Constant constant) { + constants.put(constant.name(), constant); } - /** Returns an unmodifiable view of the constants available in this */ - public Map<String, Constant> getConstants() { - if (inherited().isEmpty()) return new HashMap<>(constants); + /** Returns an unmodifiable view of the constants declared in this */ + public Map<Reference, Constant> declaredConstants() { return Collections.unmodifiableMap(constants); } - Map<String, Constant> allConstants = new HashMap<>(); + /** Returns an unmodifiable view of the constants available in this */ + public Map<Reference, Constant> constants() { + Map<Reference, Constant> allConstants = new HashMap<>(); for (var inheritedProfile : inherited()) { - for (var constant : inheritedProfile.getConstants().entrySet()) { - if (allConstants.containsKey(constant.getKey())) - throw new IllegalArgumentException("Constant '" + constant.getKey() + "' is present in " + + for (var constant : inheritedProfile.constants().values()) { + if (allConstants.containsKey(constant.name())) + throw new IllegalArgumentException(constant + "' is present in " + inheritedProfile + " inherited by " + this + ", but is also present in another profile inherited by it"); - allConstants.put(constant.getKey(), constant.getValue()); + allConstants.put(constant.name(), constant); } } + + if (schema != null) + allConstants.putAll(schema.constants()); allConstants.putAll(constants); return allConstants; } + public void add(OnnxModel model) { + onnxModels.put(model.getName(), model); + } + + /** Returns an unmodifiable map of the onnx models declared in this. */ + public Map<String, OnnxModel> declaredOnnxModels() { return onnxModels; } + + /** Returns an unmodifiable map of the onnx models available in this. */ + public Map<String, OnnxModel> onnxModels() { + Map<String, OnnxModel> allModels = new HashMap<>(); + for (var inheritedProfile : inherited()) { + for (var model : inheritedProfile.onnxModels().values()) { + if (allModels.containsKey(model.getName())) + throw new IllegalArgumentException(model + "' is present in " + + inheritedProfile + " inherited by " + + this + ", but is also present in another profile inherited by it"); + allModels.put(model.getName(), model); + } + } + + if (schema != null) + allModels.putAll(schema.onnxModels()); + allModels.putAll(onnxModels); + return allModels; + } + public void addAttributeType(String attributeName, String attributeType) { attributeTypes.addType(attributeName, attributeType); } @@ -943,7 +958,7 @@ public class RankProfile implements Cloneable { } private void compileThis(QueryProfileRegistry queryProfiles, ImportedMlModels importedModels) { - checkNameCollisions(getFunctions(), getConstants()); + checkNameCollisions(getFunctions(), constants()); ExpressionTransforms expressionTransforms = new ExpressionTransforms(); Map<Reference, TensorType> featureTypes = featureTypes(); @@ -951,8 +966,8 @@ public class RankProfile implements Cloneable { Map<String, RankingExpressionFunction> inlineFunctions = compileFunctions(this::getInlineFunctions, queryProfiles, featureTypes, importedModels, Collections.emptyMap(), expressionTransforms); - firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); - secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, getConstants(), inlineFunctions, expressionTransforms); + firstPhaseRanking = compile(this.getFirstPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); + secondPhaseRanking = compile(this.getSecondPhase(), queryProfiles, featureTypes, importedModels, constants(), inlineFunctions, expressionTransforms); // Function compiling second pass: compile all functions and insert previously compiled inline functions // TODO: This merges all functions from inherited profiles too and erases inheritance information. Not good. @@ -960,9 +975,9 @@ public class RankProfile implements Cloneable { allFunctionsCached = null; } - private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<String, Constant> constants) { - for (Map.Entry<String, RankingExpressionFunction> functionEntry : functions.entrySet()) { - if (constants.containsKey(functionEntry.getKey())) + private void checkNameCollisions(Map<String, RankingExpressionFunction> functions, Map<Reference, Constant> constants) { + for (var functionEntry : functions.entrySet()) { + if (constants.containsKey(FeatureNames.asConstantFeature(functionEntry.getKey()))) throw new IllegalArgumentException("Cannot have both a constant and function named '" + functionEntry.getKey() + "'"); } @@ -987,7 +1002,7 @@ public class RankProfile implements Cloneable { while (null != (entry = findUncompiledFunction(functions.get(), compiledFunctions.keySet()))) { RankingExpressionFunction rankingExpressionFunction = entry.getValue(); RankingExpressionFunction compiled = compile(rankingExpressionFunction, queryProfiles, featureTypes, - importedModels, getConstants(), inlineFunctions, + importedModels, constants(), inlineFunctions, expressionTransforms); compiledFunctions.put(entry.getKey(), compiled); } @@ -1007,7 +1022,7 @@ public class RankProfile implements Cloneable { QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, - Map<String, Constant> constants, + Map<Reference, Constant> constants, Map<String, RankingExpressionFunction> inlineFunctions, ExpressionTransforms expressionTransforms) { if (function == null) return null; @@ -1047,9 +1062,7 @@ public class RankProfile implements Cloneable { Map<Reference, TensorType> featureTypes) { MapEvaluationTypeContext context = new MapEvaluationTypeContext(getExpressionFunctions(), featureTypes); - // Add small and large constants, respectively - getConstants().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.type())); - rankingConstants().asMap().forEach((k, v) -> context.setType(FeatureNames.asConstantFeature(k), v.getTensorType())); + constants().forEach((k, v) -> context.setType(k, v.type())); // Add query features from all rank profile types for (QueryProfileType queryProfileType : queryProfiles.getTypeRegistry().allComponents()) { @@ -1071,10 +1084,8 @@ public class RankProfile implements Cloneable { } // Add output types for ONNX models - for (Map.Entry<String, OnnxModel> entry : onnxModels().entrySet()) { - String modelName = entry.getKey(); - OnnxModel model = entry.getValue(); - Arguments args = new Arguments(new ReferenceNode(modelName)); + for (var model : onnxModels().values()) { + Arguments args = new Arguments(new ReferenceNode(model.getName())); Map<String, TensorType> inputTypes = resolveOnnxInputTypes(model, context); TensorType defaultOutputType = model.getTensorType(model.getDefaultOutput(), inputTypes); @@ -1428,23 +1439,32 @@ public class RankProfile implements Cloneable { private final Optional<Tensor> value; private final Optional<String> valuePath; + // Always set only if valuePath is set + private final Optional<DistributableResource.PathType> pathType; + public Constant(Reference name, Tensor value) { - this(name, value.type(), Optional.of(value), Optional.empty()); + this(name, value.type(), Optional.of(value), Optional.empty(), Optional.empty()); } public Constant(Reference name, TensorType type, String valuePath) { - this(name, type, Optional.empty(), Optional.of(valuePath)); + this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(DistributableResource.PathType.FILE)); } - private Constant(Reference name, TensorType type, Optional<Tensor> value, Optional<String> valuePath) { + public Constant(Reference name, TensorType type, String valuePath, DistributableResource.PathType pathType) { + this(name, type, Optional.empty(), Optional.of(valuePath), Optional.of(pathType)); + } + + private Constant(Reference name, TensorType type, Optional<Tensor> value, + Optional<String> valuePath, Optional<DistributableResource.PathType> pathType) { + this.name = Objects.requireNonNull(name); + this.type = Objects.requireNonNull(type); + this.value = Objects.requireNonNull(value); + this.valuePath = Objects.requireNonNull(valuePath); + this.pathType = Objects.requireNonNull(pathType); + if (type.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) throw new IllegalArgumentException("Illegal type of constant " + name + " type " + type + ": Dense tensor dimensions must have a size"); - - this.name = name; - this.type = type; - this.value = value; - this.valuePath = valuePath; } public Reference name() { return name; } @@ -1456,6 +1476,9 @@ public class RankProfile implements Cloneable { /** Returns the path to the value of this, if its value is empty. */ public Optional<String> valuePath() { return valuePath; } + /** Returns the path type, if valuePath is set. */ + public Optional<DistributableResource.PathType> pathType() { return pathType; } + @Override public boolean equals(Object o) { if (o == this) return true; @@ -1465,18 +1488,19 @@ public class RankProfile implements Cloneable { if ( ! other.type().equals(this.type())) return false; if ( ! other.value().equals(this.value())) return false; if ( ! other.valuePath().equals(this.valuePath())) return false; + if ( ! other.pathType().equals(this.pathType())) return false; return true; } @Override public int hashCode() { - return Objects.hash(name, type, value, valuePath); + return Objects.hash(name, type, value, valuePath, pathType); } @Override public String toString() { - return "constant '" + name + "' " + type + - (value().isPresent() ? ":" + value.get().toAbbreviatedString() : "file:" + valuePath.get()); + return "constant '" + name + "' " + type + ":" + + (value().isPresent() ? value.get().toAbbreviatedString() : " file:" + valuePath.get()); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java index bb2dc04cd2a..06ffc934b2d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankProfileRegistry.java +++ b/config-model/src/main/java/com/yahoo/schema/RankProfileRegistry.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; import java.util.ArrayList; import java.util.Arrays; @@ -31,8 +31,8 @@ public class RankProfileRegistry { public static RankProfileRegistry createRankProfileRegistryWithBuiltinRankProfiles(Schema schema) { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); return rankProfileRegistry; } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java b/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java index ce32f323f22..64bca9367d8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/SDDocumentTypeOrderer.java +++ b/config-model/src/main/java/com/yahoo/schema/SDDocumentTypeOrderer.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.TemporarySDDocumentType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.TemporarySDDocumentType; import java.util.*; import java.util.logging.Level; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java b/config-model/src/main/java/com/yahoo/schema/Schema.java index 203a109acd5..c733b6012f9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/Schema.java +++ b/config-model/src/main/java/com/yahoo/schema/Schema.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -9,16 +9,17 @@ import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DataTypeName; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.derived.SummaryClass; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.document.TemporaryImportedFields; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.document.TemporaryImportedFields; +import com.yahoo.schema.document.annotation.SDAnnotationType; +import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; @@ -85,9 +86,12 @@ public class Schema implements ImmutableSchema { /** External rank expression files of this */ private final LargeRankExpressions largeRankExpressions; - private final RankingConstants rankingConstants; + /** Constants that will be available in all rank profiles. */ + // TODO: Remove on Vespa 9: Should always be in a rank profile + private final Map<Reference, RankProfile.Constant> constants = new LinkedHashMap<>(); - private final OnnxModels onnxModels; + // TODO: Remove on Vespa 9: Should always be in a rank profile + private final Map<String, OnnxModel> onnxModels = new LinkedHashMap<>(); /** All imported fields of this (and parent schemas) */ // TODO: Use empty, not optional @@ -147,8 +151,6 @@ public class Schema implements ImmutableSchema { this.properties = properties; this.documentsOnly = documentsOnly; largeRankExpressions = new LargeRankExpressions(fileRegistry); - rankingConstants = new RankingConstants(fileRegistry, Optional.of(this)); - onnxModels = new OnnxModels(fileRegistry, Optional.of(this)); } /** @@ -216,7 +218,7 @@ public class Schema implements ImmutableSchema { */ public void addDocument(SDDocumentType document) { if (documentType != null) { - throw new IllegalArgumentException("Searchdefinition cannot have more than one document"); + throw new IllegalArgumentException("Schema cannot have more than one document"); } documentType = document; } @@ -224,11 +226,41 @@ public class Schema implements ImmutableSchema { @Override public LargeRankExpressions rankExpressionFiles() { return largeRankExpressions; } + public void add(RankProfile.Constant constant) { + constants.put(constant.name(), constant); + } + + /** Returns an unmodifiable map of the constants declared in this. */ + public Map<Reference, RankProfile.Constant> declaredConstants() { return constants; } + + /** Returns an unmodifiable map of the constants available in this. */ @Override - public RankingConstants rankingConstants() { return rankingConstants; } + public Map<Reference, RankProfile.Constant> constants() { + if (inherited().isEmpty()) return Collections.unmodifiableMap(constants); + if (constants.isEmpty()) return inherited().get().constants(); + + Map<Reference, RankProfile.Constant> allConstants = new LinkedHashMap<>(inherited().get().constants()); + allConstants.putAll(constants); + return allConstants; + } + public void add(OnnxModel model) { + onnxModels.put(model.getName(), model); + } + + /** Returns an unmodifiable map of the onnx models declared in this. */ + public Map<String, OnnxModel> declaredOnnxModels() { return onnxModels; } + + /** Returns an unmodifiable map of the onnx models available in this. */ @Override - public OnnxModels onnxModels() { return onnxModels; } + public Map<String, OnnxModel> onnxModels() { + if (inherited().isEmpty()) return Collections.unmodifiableMap(onnxModels); + if (onnxModels.isEmpty()) return inherited().get().onnxModels(); + + Map<String, OnnxModel> allModels = new LinkedHashMap<>(inherited().get().onnxModels()); + allModels.putAll(onnxModels); + return allModels; + } public Optional<TemporaryImportedFields> temporaryImportedFields() { return temporaryImportedFields; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/TemporarySDTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java index 2eaf0d5e5ba..b1ce6f5eb4f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/TemporarySDTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/TemporarySDTypeResolver.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.TemporarySDDocumentType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.TemporarySDDocumentType; import java.util.Collection; import java.util.LinkedList; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/UnrankedRankProfile.java index acf034362ca..6c1f5fc8731 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/UnrankedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/UnrankedRankProfile.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.parser.ParseException; @@ -11,8 +11,8 @@ import com.yahoo.searchlib.rankingexpression.parser.ParseException; */ public class UnrankedRankProfile extends RankProfile { - public UnrankedRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry, RankingConstants rankingConstants) { - super("unranked", schema, rankProfileRegistry, rankingConstants); + public UnrankedRankProfile(Schema schema, RankProfileRegistry rankProfileRegistry) { + super("unranked", schema, rankProfileRegistry); try { RankingExpression exp = new RankingExpression("value(0)"); this.setFirstPhaseRanking(exp); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java index cd10263b0ff..99f73a75669 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/AttributeFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/AttributeFields.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.subscription.ConfigInstanceUtil; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Ranking; -import com.yahoo.searchdefinition.document.Sorting; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Ranking; +import com.yahoo.schema.document.Sorting; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.indexinglanguage.expressions.ToPositionExpression; @@ -21,10 +21,10 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedComplexField; /** * The set of all attribute fields defined by a search definition diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java index 99c40c2700f..9943a02a2f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Derived.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Derived.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.ConfigInstance; import com.yahoo.config.ConfigInstance.Builder; import com.yahoo.document.Field; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.text.StringUtilities; import java.io.IOException; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java index 0c7c507f32d..0f5721bbab3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/DerivedConfiguration.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/DerivedConfiguration.java @@ -1,27 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.ConfigInstance; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.model.deploy.TestProperties; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; import com.yahoo.io.IOUtils; import com.yahoo.protect.Validator; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.validation.Validation; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.validation.Validation; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.io.IOException; import java.io.Writer; -import java.util.concurrent.ExecutorService; /** * A set of all derived configuration of a schema. Use this as a facade to individual configurations when @@ -55,54 +50,42 @@ public class DerivedConfiguration implements AttributesConfig.Producer { * @param schema the search to derive a configuration from. Derived objects will be snapshots, but this argument is * live. Which means that this object will be inconsistent when the given search definition is later * modified. - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param rankProfileRegistry a {@link com.yahoo.schema.RankProfileRegistry} */ public DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry) { this(schema, rankProfileRegistry, new QueryProfileRegistry()); } DerivedConfiguration(Schema schema, RankProfileRegistry rankProfileRegistry, QueryProfileRegistry queryProfiles) { - this(schema, new BaseDeployLogger(), new TestProperties(), rankProfileRegistry, queryProfiles, new ImportedMlModels(), new InThreadExecutorService()); + this(schema, new DeployState.Builder().rankProfileRegistry(rankProfileRegistry).queryProfiles(queryProfiles).build()); } /** - * Creates a complete derived configuration snapshot from a search definition. + * Creates a complete derived configuration snapshot from a schema. * - * @param schema the search to derive a configuration from. Derived objects will be snapshots, but this - * argument is live. Which means that this object will be inconsistent when the given - * search definition is later modified. - * @param deployLogger a {@link DeployLogger} for logging when doing operations on this - * @param deployProperties properties set on deploy - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} - * @param queryProfiles the query profiles of this application + * @param schema the schema to derive a configuration from. Derived objects will be snapshots, but this + * argument is live. Which means that this object will be inconsistent if the given + * schema is later modified. */ - public DerivedConfiguration(Schema schema, - DeployLogger deployLogger, - ModelContext.Properties deployProperties, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - ExecutorService executor) { - Validator.ensureNotNull("Search definition", schema); + public DerivedConfiguration(Schema schema, DeployState deployState) { + Validator.ensureNotNull("Schema", schema); this.schema = schema; - this.queryProfiles = queryProfiles; - this.maxUncommittedMemory = deployProperties.featureFlags().maxUnCommittedMemory(); - this.enableBitVectors = deployProperties.featureFlags().enableBitVectors(); + this.queryProfiles = deployState.getQueryProfiles().getRegistry(); + this.maxUncommittedMemory = deployState.getProperties().featureFlags().maxUnCommittedMemory(); + this.enableBitVectors = deployState.getProperties().featureFlags().enableBitVectors(); if ( ! schema.isDocumentsOnly()) { streamingFields = new VsmFields(schema); streamingSummary = new VsmSummary(schema); } if ( ! schema.isDocumentsOnly()) { attributeFields = new AttributeFields(schema); - summaries = new Summaries(schema, deployLogger, deployProperties.featureFlags()); + summaries = new Summaries(schema, deployState.getDeployLogger(), deployState.getProperties().featureFlags()); summaryMap = new SummaryMap(schema); juniperrc = new Juniperrc(schema); - rankProfileList = new RankProfileList(schema, schema.rankingConstants(), schema.rankExpressionFiles(), - schema.onnxModels(), attributeFields, rankProfileRegistry, - queryProfiles, importedModels, deployProperties, executor); + rankProfileList = new RankProfileList(schema, schema.rankExpressionFiles(), attributeFields, deployState); indexingScript = new IndexingScript(schema); indexInfo = new IndexInfo(schema); - schemaInfo = new SchemaInfo(schema, rankProfileRegistry, summaries, summaryMap); + schemaInfo = new SchemaInfo(schema, deployState.rankProfileRegistry(), summaries, summaryMap); indexSchema = new IndexSchema(schema); importedFields = new ImportedFields(schema); } @@ -144,6 +127,12 @@ public class DerivedConfiguration implements AttributesConfig.Producer { exportCfg(new QueryProfiles(queryProfileRegistry, (level, message) -> {}).getConfig(), toDirectory + "/" + "query-profiles.cfg"); } + public void exportConstants(String toDirectory) throws IOException { + RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); + rankProfileList.getConfig(b); + exportCfg(b.build(), toDirectory + "/" + "ranking-constants.cfg"); + } + private static void exportCfg(ConfigInstance instance, String fileName) throws IOException { Writer writer = null; try { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java index c90855896ec..44bea43a8e3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Deriver.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Deriver.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import java.io.IOException; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java b/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java index ee96161d443..4fccfb5d9f8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Exportable.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Exportable.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; /** * Classes exportable to configurations diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java b/config-model/src/main/java/com/yahoo/schema/derived/FieldRankSettings.java index 2a2950f71e9..ccb25df031c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldRankSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FieldRankSettings.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.collections.Pair; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java b/config-model/src/main/java/com/yahoo/schema/derived/FieldResultTransform.java index 39432fd0049..99b2925d714 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/FieldResultTransform.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/FieldResultTransform.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java new file mode 100644 index 00000000000..05f6be2f6f1 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedConstants.java @@ -0,0 +1,87 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.schema.DistributableResource; +import com.yahoo.schema.RankProfile; +import com.yahoo.tensor.TensorType; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Constant values for ranking/model execution tied to a rank profile, + * to be distributed as files. + * + * @author bratseth + */ +public class FileDistributedConstants { + + private final Map<String, DistributableConstant> constants; + + public FileDistributedConstants(FileRegistry fileRegistry, Collection<RankProfile.Constant> constants) { + Map<String, DistributableConstant> distributableConstants = new LinkedHashMap<>(); + for (var constant : constants) { + if ( ! constant.valuePath().isPresent()) continue; + + var distributableConstant = new DistributableConstant(constant.name().simpleArgument().get(), + constant.type(), + constant.valuePath().get(), + constant.pathType().get()); + distributableConstant.validate(); + distributableConstant.register(fileRegistry); + distributableConstants.put(distributableConstant.getName(), distributableConstant); + } + this.constants = Collections.unmodifiableMap(distributableConstants); + } + + /** Returns a read-only map of the constants in this indexed by name. */ + public Map<String, DistributableConstant> asMap() { return constants; } + + public void getConfig(RankingConstantsConfig.Builder builder) { + for (var constant : constants.values()) { + builder.constant(new RankingConstantsConfig.Constant.Builder() + .name(constant.getName()) + .fileref(constant.getFileReference()) + .type(constant.getType())); + } + } + + public static class DistributableConstant extends DistributableResource { + + private final TensorType tensorType; + + public DistributableConstant(String name, TensorType type, String fileName) { + this(name, type, fileName, PathType.FILE); + } + + public DistributableConstant(String name, TensorType type, String fileName, PathType pathType) { + super(name, fileName, pathType); + this.tensorType = type; + validate(); + } + + public TensorType getTensorType() { return tensorType; } + public String getType() { return tensorType.toString(); } + + public void validate() { + super.validate(); + if (tensorType == null) + throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); + if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) + throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + + ": Dense tensor dimensions must have a size"); + } + + @Override + public String toString() { + return super.toString() + "' of type '" + tensorType + "'"; + } + + } + +} + diff --git a/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java new file mode 100644 index 00000000000..b5c3909c78c --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/FileDistributedOnnxModels.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import com.yahoo.config.application.api.FileRegistry; +import com.yahoo.schema.OnnxModel; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.logging.Logger; + +/** + * ONNX models distributed as files. + * + * @author bratseth + */ +public class FileDistributedOnnxModels { + + private static final Logger log = Logger.getLogger(FileDistributedOnnxModels.class.getName()); + + private final Map<String, OnnxModel> models; + + public FileDistributedOnnxModels(FileRegistry fileRegistry, Collection<OnnxModel> models) { + Map<String, OnnxModel> distributableModels = new LinkedHashMap<>(); + for (var model : models) { + model.validate(); + model.register(fileRegistry); + distributableModels.put(model.getName(), model); + } + this.models = Collections.unmodifiableMap(distributableModels); + } + + public Map<String, OnnxModel> asMap() { return models; } + + public void getConfig(OnnxModelsConfig.Builder builder) { + for (OnnxModel model : models.values()) { + if ("".equals(model.getFileReference())) + log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way + else { + OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); + modelBuilder.dry_run_on_setup(true); + modelBuilder.name(model.getName()); + modelBuilder.fileref(model.getFileReference()); + model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); + model.getOutputMap().forEach((name, as) -> modelBuilder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); + if (model.getStatelessExecutionMode().isPresent()) + modelBuilder.stateless_execution_mode(model.getStatelessExecutionMode().get()); + if (model.getStatelessInterOpThreads().isPresent()) + modelBuilder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); + if (model.getStatelessIntraOpThreads().isPresent()) + modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); + + builder.model(modelBuilder); + } + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java index 3b8c0a9cff2..fa3f49f06d5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/ImportedFields.java @@ -1,20 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedComplexField; -import com.yahoo.searchdefinition.document.ImportedField; +package com.yahoo.schema.derived; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; import com.yahoo.vespa.config.search.ImportedFieldsConfig; import java.util.Optional; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; /** * This class derives imported fields from search definition and produces imported-fields.cfg as needed by the search backend. @@ -23,7 +22,7 @@ import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isM */ public class ImportedFields extends Derived implements ImportedFieldsConfig.Producer { - private Optional<com.yahoo.searchdefinition.document.ImportedFields> importedFields = Optional.empty(); + private Optional<com.yahoo.schema.document.ImportedFields> importedFields = Optional.empty(); public ImportedFields(Schema schema) { derive(schema); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java b/config-model/src/main/java/com/yahoo/schema/derived/Index.java index c19fc9ca942..3b5e617d3dc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Index.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Index.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java index b65af3a8f02..4887ad52974 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexInfo.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; @@ -8,19 +8,19 @@ import com.yahoo.document.MapDataType; import com.yahoo.document.NumericDataType; import com.yahoo.document.PrimitiveDataType; import com.yahoo.document.StructuredDataType; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.processing.ExactMatch; -import com.yahoo.searchdefinition.processing.NGramMatch; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.processing.ExactMatch; +import com.yahoo.schema.processing.NGramMatch; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.search.config.IndexInfoConfig; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java index ce5731bcdeb..7f6c824b979 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexSchema.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; @@ -7,10 +7,10 @@ import com.yahoo.document.Field; import com.yahoo.document.StructuredDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.config.search.IndexschemaConfig; import java.util.ArrayList; @@ -75,7 +75,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { private void deriveIndexFields(Field field, Schema schema) { IndexField toAdd = new IndexField(field.getName(), Index.convertType(field.getDataType()), field.getDataType()); - com.yahoo.searchdefinition.Index definedIndex = schema.getIndex(field.getName()); + com.yahoo.schema.Index definedIndex = schema.getIndex(field.getName()); if (definedIndex != null) { toAdd.setIndexSettings(definedIndex); } @@ -174,7 +174,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { public static class IndexField { private String name; private Index.Type type; - private com.yahoo.searchdefinition.Index.Type sdType; // The index type in "user intent land" + private com.yahoo.schema.Index.Type sdType; // The index type in "user intent land" private DataType sdFieldType; private boolean prefix = false; private boolean phrases = false; // TODO dead, but keep a while to ensure config compatibility? @@ -188,7 +188,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { this.type = type; this.sdFieldType = sdFieldType; } - public void setIndexSettings(com.yahoo.searchdefinition.Index index) { + public void setIndexSettings(com.yahoo.schema.Index index) { if (type.equals(Index.Type.TEXT)) { prefix = index.isPrefix(); interleavedFeatures = index.useInterleavedFeatures(); @@ -224,7 +224,7 @@ public class IndexSchema extends Derived implements IndexschemaConfig.Producer { * The user set index type * @return the type */ - public com.yahoo.searchdefinition.Index.Type getSdType() { + public com.yahoo.schema.Index.Type getSdType() { return sdType; } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java index 23409729dbb..6dae89bf692 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/IndexingScript.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/IndexingScript.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig.Ilscript.Builder; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java index 327d7df4d07..162efbb25b4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Juniperrc.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Juniperrc.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.config.search.summary.JuniperrcConfig; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinition.java index 66526684dc4..7d558ea51cc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinition.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinition.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.schema.document.RankType; import java.util.Collections; import java.util.Iterator; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinitionSet.java index cadd3a9efb0..65e68181b5b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionSet.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/NativeRankTypeDefinitionSet.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.schema.document.RankType; import java.util.Collections; import java.util.Map; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java b/config-model/src/main/java/com/yahoo/schema/derived/NativeTable.java index 3d1f565b793..6eff2487bca 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/NativeTable.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/NativeTable.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; /** * A named rank table of a certain type. diff --git a/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java new file mode 100644 index 00000000000..c549f9af7ce --- /dev/null +++ b/config-model/src/main/java/com/yahoo/schema/derived/RankProfileList.java @@ -0,0 +1,210 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.schema.derived; + +import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.config.model.api.ModelContext; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.search.query.profile.QueryProfileRegistry; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.searchlib.rankingexpression.Reference; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; +import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +/** + * The derived rank profiles of a schema + * + * @author bratseth + */ +public class RankProfileList extends Derived implements RankProfilesConfig.Producer { + + private final Map<String, RawRankProfile> rankProfiles; + private final FileDistributedConstants constants; + private final LargeRankExpressions largeRankExpressions; + private final FileDistributedOnnxModels onnxModels; + + public static final RankProfileList empty = new RankProfileList(); + + private RankProfileList() { + constants = new FileDistributedConstants(null, List.of()); + largeRankExpressions = new LargeRankExpressions(null); + onnxModels = new FileDistributedOnnxModels(null, List.of()); + rankProfiles = Map.of(); + } + + /** + * Creates a rank profile list + * + * @param schema the schema this is a rank profile from + * @param attributeFields the attribute fields to create a ranking for + */ + public RankProfileList(Schema schema, + LargeRankExpressions largeRankExpressions, + AttributeFields attributeFields, + DeployState deployState) { + setName(schema == null ? "default" : schema.getName()); + this.largeRankExpressions = largeRankExpressions; + this.rankProfiles = deriveRankProfiles(schema, attributeFields, deployState); + this.constants = deriveFileDistributedConstants(schema, rankProfiles.values(), deployState); + this.onnxModels = deriveFileDistributedOnnxModels(schema, rankProfiles.values(), deployState); + } + + private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry, Set<String> processedProfiles) { + return rank.inheritedNames().isEmpty() || + processedProfiles.containsAll(rank.inheritedNames()) || + (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); + } + + private Map<String, RawRankProfile> deriveRankProfiles(Schema schema, + AttributeFields attributeFields, + DeployState deployState) { + Map<String, RawRankProfile> rawRankProfiles = new LinkedHashMap<>(); + if (schema != null) { // profiles belonging to a schema have a default profile + RawRankProfile rawRank = new RawRankProfile(deployState.rankProfileRegistry().get(schema, "default"), + largeRankExpressions, + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + attributeFields, + deployState.getProperties()); + rawRankProfiles.put(rawRank.getName(), rawRank); + } + + Map<String, RankProfile> remaining = new LinkedHashMap<>(); + deployState.rankProfileRegistry().rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); + remaining.remove("default"); + while (!remaining.isEmpty()) { + List<RankProfile> ready = new ArrayList<>(); + remaining.forEach((name, profile) -> { + if (areDependenciesReady(profile, deployState.rankProfileRegistry(), rawRankProfiles.keySet())) + ready.add(profile); + }); + rawRankProfiles.putAll(processRankProfiles(ready, + deployState.getQueryProfiles().getRegistry(), + deployState.getImportedModels(), + attributeFields, + deployState.getProperties(), + deployState.getExecutor())); + ready.forEach(rank -> remaining.remove(rank.name())); + } + return rawRankProfiles; + } + + private Map<String, RawRankProfile> processRankProfiles(List<RankProfile> profiles, + QueryProfileRegistry queryProfiles, + ImportedMlModels importedModels, + AttributeFields attributeFields, + ModelContext.Properties deployProperties, + ExecutorService executor) { + Map<String, Future<RawRankProfile>> futureRawRankProfiles = new LinkedHashMap<>(); + for (RankProfile profile : profiles) { + futureRawRankProfiles.put(profile.name(), executor.submit(() -> new RawRankProfile(profile, largeRankExpressions, queryProfiles, importedModels, + attributeFields, deployProperties))); + } + try { + Map<String, RawRankProfile> rawRankProfiles = new LinkedHashMap<>(); + for (Future<RawRankProfile> rawFuture : futureRawRankProfiles.values()) { + RawRankProfile rawRank = rawFuture.get(); + rawRankProfiles.put(rawRank.getName(), rawRank); + } + return rawRankProfiles; + } catch (InterruptedException | ExecutionException e) { + throw new IllegalStateException(e); + } + } + + private static FileDistributedConstants deriveFileDistributedConstants(Schema schema, + Collection<RawRankProfile> rankProfiles, + DeployState deployState) { + Map<Reference, RankProfile.Constant> allFileConstants = new HashMap<>(); + addFileConstants(schema != null ? schema.constants().values() : List.of(), + allFileConstants, + schema != null ? schema.toString() : "[global]"); + for (var profile : rankProfiles) + addFileConstants(profile.constants(), allFileConstants, profile.toString()); + return new FileDistributedConstants(deployState.getFileRegistry(), allFileConstants.values()); + } + + private static void addFileConstants(Collection<RankProfile.Constant> source, + Map<Reference, RankProfile.Constant> destination, + String sourceName) { + for (var constant : source) { + if (constant.valuePath().isEmpty()) continue; + var existing = destination.get(constant.name()); + if ( existing != null && ! constant.equals(existing)) { + throw new IllegalArgumentException("Duplicate constants: " + sourceName + " have " + constant + + ", but we already have " + existing + + ": Value reference constants must be unique across all rank profiles/models"); + } + destination.put(constant.name(), constant); + } + } + + private static FileDistributedOnnxModels deriveFileDistributedOnnxModels(Schema schema, + Collection<RawRankProfile> rankProfiles, + DeployState deployState) { + Map<String, OnnxModel> allModels = new LinkedHashMap<>(); + addOnnxModels(schema != null ? schema.onnxModels().values() : List.of(), + allModels, + schema != null ? schema.toString() : "[global]"); + for (var profile : rankProfiles) + addOnnxModels(profile.onnxModels(), allModels, profile.toString()); + return new FileDistributedOnnxModels(deployState.getFileRegistry(), allModels.values()); + } + + private static void addOnnxModels(Collection<OnnxModel> source, + Map<String, OnnxModel> destination, + String sourceName) { + for (var model : source) { + var existing = destination.get(model.getName()); + if ( existing != null && ! model.equals(existing)) { + throw new IllegalArgumentException("Duplicate onnx model: " + sourceName + " have " + model + + ", but we already have " + existing + + ": Onnx models must be unique across all rank profiles/models"); + } + destination.put(model.getName(), model); + } + } + + public Map<String, RawRankProfile> getRankProfiles() { return rankProfiles; } + public FileDistributedConstants constants() { return constants; } + public FileDistributedOnnxModels getOnnxModels() { return onnxModels; } + + @Override + public String getDerivedName() { return "rank-profiles"; } + + @Override + public void getConfig(RankProfilesConfig.Builder builder) { + for (RawRankProfile rank : rankProfiles.values() ) { + rank.getConfig(builder); + } + } + + public void getConfig(RankingExpressionsConfig.Builder builder) { + largeRankExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); + } + + public void getConfig(RankingConstantsConfig.Builder builder) { + constants.getConfig(builder); + } + + public void getConfig(OnnxModelsConfig.Builder builder) { + onnxModels.getConfig(builder); + } + +} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java index 77245da5ddd..66389a030af 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RawRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/RawRankProfile.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.google.common.collect.ImmutableList; @@ -7,13 +7,13 @@ import com.yahoo.collections.Pair; import com.yahoo.compress.Compressor; import com.yahoo.config.model.api.ModelContext; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankExpressionBody; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankExpressionBody; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.expressiontransforms.OnnxModelTransformer; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; @@ -26,6 +26,7 @@ import com.yahoo.vespa.config.search.RankProfilesConfig; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.LinkedHashMap; @@ -39,6 +40,7 @@ import java.util.stream.Collectors; /** * A rank profile derived from a search definition, containing exactly the features available natively in the server + * RawRankProfile has a long lifetime so do not refer objects not necessary. * * @author bratseth */ @@ -53,15 +55,29 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private final String name; private final Compressor.Compression compressedProperties; + /** The compiled profile this is created from. */ + private final Collection<RankProfile.Constant> constants; + private final Collection<OnnxModel> onnxModels; + /** Creates a raw rank profile from the given rank profile. */ public RawRankProfile(RankProfile rankProfile, LargeRankExpressions largeExpressions, QueryProfileRegistry queryProfiles, ImportedMlModels importedModels, AttributeFields attributeFields, ModelContext.Properties deployProperties) { this.name = rankProfile.name(); - compressedProperties = compress(new Deriver(rankProfile.compile(queryProfiles, importedModels), attributeFields, deployProperties, queryProfiles) + /* + * Forget the RankProfiles as soon as possible. They can become very large and memory hungry + * Especially do not refer then through any member variables due to the RawRankProfile living forever. + */ + RankProfile compiled = rankProfile.compile(queryProfiles, importedModels); + constants = compiled.constants().values(); + onnxModels = compiled.onnxModels().values(); + compressedProperties = compress(new Deriver(compiled, attributeFields, deployProperties, queryProfiles) .derive(largeExpressions)); } + public Collection<RankProfile.Constant> constants() { return constants; } + public Collection<OnnxModel> onnxModels() { return onnxModels; } + private Compressor.Compression compress(List<Pair<String, String>> properties) { StringBuilder b = new StringBuilder(); for (Pair<String, String> property : properties) @@ -426,15 +442,18 @@ public class RawRankProfile implements RankProfilesConfig.Producer { for (Map.Entry<String, String> attributeType : attributeTypes.entrySet()) { properties.add(new Pair<>("vespa.type.attribute." + attributeType.getKey(), attributeType.getValue())); } + for (var input : inputs.values()) { if (FeatureNames.isQueryFeature(input.name())) { - properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), - input.type().toString())); - if (input.defaultValue().isPresent()) + if (input.type().rank() > 0) // Proton does not like representing the double type as a rank 0 tensor + properties.add(new Pair<>("vespa.type.query." + input.name().arguments().expressions().get(0), + input.type().toString())); + if (input.defaultValue().isPresent()) { properties.add(new Pair<>(input.name().toString(), input.type().rank() == 0 ? String.valueOf(input.defaultValue().get().asDouble()) : - input.defaultValue().get().toString(false, false))); + input.defaultValue().get().toString(true, false))); + } } } if (properties.size() >= 1000000) throw new IllegalArgumentException("Too many rank properties"); @@ -477,7 +496,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void deriveOnnxModelFunctionsAndFeatures(RankProfile rankProfile) { if (rankProfile.schema() == null) return; - if (rankProfile.schema().onnxModels().asMap().isEmpty()) return; + if (rankProfile.onnxModels().isEmpty()) return; replaceOnnxFunctionInputs(rankProfile); replaceImplicitOnnxConfigFeatures(summaryFeatures, rankProfile); replaceImplicitOnnxConfigFeatures(matchFeatures, rankProfile); @@ -486,7 +505,7 @@ public class RawRankProfile implements RankProfilesConfig.Producer { private void replaceOnnxFunctionInputs(RankProfile rankProfile) { Set<String> functionNames = rankProfile.getFunctions().keySet(); if (functionNames.isEmpty()) return; - for (OnnxModel onnxModel: rankProfile.schema().onnxModels().asMap().values()) { + for (OnnxModel onnxModel: rankProfile.onnxModels().values()) { for (Map.Entry<String, String> mapping : onnxModel.getInputMap().entrySet()) { String source = mapping.getValue(); if (functionNames.contains(source)) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java index 0c1e5a76a89..18c6f335787 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SchemaInfo.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SchemaInfo.java @@ -1,22 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.tensor.Tensor; -import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.documentmodel.SummaryTransform; -import com.yahoo.vespa.model.search.SearchCluster; import java.util.Collection; import java.util.Collections; import java.util.LinkedHashMap; import java.util.Map; -import java.util.Optional; /** * Information about a schema. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java index 8703eb11433..3bab808beff 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SearchOrderer.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SearchOrderer.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.DataTypeName; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java index e259c07dbb5..2b41fbb3b1a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/Summaries.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/Summaries.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.config.search.SummaryConfig; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java index ea8e2aac1f0..193c6893203 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClass.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClass.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; import com.yahoo.prelude.fastsearch.DocsumDefinitionSet; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java index e9dd2ac321c..f042054a0b5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryClassField.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryClassField.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java index 9e1740b4073..df9174a12ed 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/SummaryMap.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/SummaryMap.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import java.util.Collections; -import java.util.Iterator; import java.util.Map; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java index c9ad8ad2838..c8679b6166c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmFields.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmFields.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; @@ -11,14 +11,14 @@ import com.yahoo.document.datatypes.PredicateFieldValue; import com.yahoo.document.datatypes.Raw; import com.yahoo.document.datatypes.StringFieldValue; import com.yahoo.document.datatypes.TensorFieldValue; -import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import java.util.LinkedHashMap; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java index 998343bc467..30ae9c97268 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/VsmSummary.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/VsmSummary.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.config.search.vsm.VsmsummaryConfig; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java b/config-model/src/main/java/com/yahoo/schema/derived/package-info.java index e908045d777..370617ac6cc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/package-info.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/package-info.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. @ExportPackage -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.osgi.annotation.ExportPackage; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java b/config-model/src/main/java/com/yahoo/schema/derived/validation/IndexStructureValidator.java index 76f52bce2a9..512d9f742bf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/IndexStructureValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/validation/IndexStructureValidator.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived.validation; +package com.yahoo.schema.derived.validation; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.derived.IndexingScript; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.IndexingScript; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validation.java index fd75fbb2f48..dba4dce49f0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validation.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived.validation; +package com.yahoo.schema.derived.validation; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; public class Validation { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validator.java index 7d3e5b8e9ed..bf0f007841c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/validation/Validator.java +++ b/config-model/src/main/java/com/yahoo/schema/derived/validation/Validator.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived.validation; +package com.yahoo.schema.derived.validation; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.DerivedConfiguration; /** * @author mathiasm diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java index 2c4a2c06d02..5593c3f8cdf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Attribute.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; @@ -232,7 +232,9 @@ public final class Attribute implements Cloneable, Serializable { public void setEnableBitVectors(boolean enableBitVectors) { this.enableBitVectors = enableBitVectors; } public void setEnableOnlyBitVector(boolean enableOnlyBitVector) { this.enableOnlyBitVector = enableOnlyBitVector; } public void setFastRank(boolean value) { - Supplier<IllegalArgumentException> badGen = () -> new IllegalArgumentException("fast-rank is only valid for tensor attributes, invalid for: "+this); + Supplier<IllegalArgumentException> badGen = () -> + new IllegalArgumentException("The " + toString() + " does not support 'fast-rank'. " + + "Only supported for tensor types with at least one mapped dimension"); var tt = tensorType.orElseThrow(badGen); for (var dim : tt.dimensions()) { if (dim.isMapped()) { @@ -425,7 +427,7 @@ public final class Attribute implements Cloneable, Serializable { @Override public String toString() { - return "attribute '" + name + "' (" + type + ")"; + return "attribute '" + name + "' (" + (tensorType.isPresent() ? tensorType.get() : type) + ")"; } public Set<String> getAliases() { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java b/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java index 3b6228029c7..8563d414c40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/BooleanIndexDefinition.java +++ b/config-model/src/main/java/com/yahoo/schema/document/BooleanIndexDefinition.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.util.Optional; import java.util.OptionalDouble; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java b/config-model/src/main/java/com/yahoo/schema/document/Case.java index ef82a498e5e..443f67940a0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Case.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Case.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; /** * Describes if items should preserve lower/upper case, or shall be uncased diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java index feac6b9618e..993bf16405a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtils.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ComplexAttributeFieldUtils.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; -import com.yahoo.document.PositionDataType; import com.yahoo.document.StructDataType; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java b/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java index 163651bf255..4744547c778 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Dictionary.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Dictionary.java @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; /** * Represents settings for dictionary control diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java b/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java index 944564dbaa1..e62e784d7b3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/FieldSet.java +++ b/config-model/src/main/java/com/yahoo/schema/document/FieldSet.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.util.LinkedHashSet; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java b/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java index 956d63a1cdf..829555d88c6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/GeoPos.java +++ b/config-model/src/main/java/com/yahoo/schema/document/GeoPos.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java b/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java index 4f717092f56..cc427356c78 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/HnswIndexParams.java +++ b/config-model/src/main/java/com/yahoo/schema/document/HnswIndexParams.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; -import java.util.Locale; import java.util.Optional; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java index bdd5a4ea899..553b5b4d940 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedComplexSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedComplexSDField.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.util.Collection; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java index 71814c31087..335942de99d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableImportedSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableImportedSDField.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java index 6d1dee67641..44e442811ba 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImmutableSDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImmutableSDField.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java index 9435f7cdd48..10c400bc4d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedComplexField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedComplexField.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; -import com.yahoo.searchdefinition.DocumentReference; +import com.yahoo.schema.DocumentReference; import java.util.Collection; import java.util.Map; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java index 6f2142c3cea..50f8591bbce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedField.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; -import com.yahoo.searchdefinition.DocumentReference; +import com.yahoo.schema.DocumentReference; /** * A field that is imported from a concrete field in a referenced document type and given an alias name. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java index 18bd3f43445..f6654896fae 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedFields.java @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; - -import com.yahoo.searchdefinition.Schema; +package com.yahoo.schema.document; import java.util.Collections; import java.util.Map; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java b/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java index 6acd1661294..244135ecc10 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/ImportedSimpleField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/ImportedSimpleField.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; -import com.yahoo.searchdefinition.DocumentReference; +import com.yahoo.schema.DocumentReference; /** * A simple field that is imported from a concrete field in a referenced document type and given an alias name. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java b/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java index 545f2e293dc..8556fe491d0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchAlgorithm.java +++ b/config-model/src/main/java/com/yahoo/schema/document/MatchAlgorithm.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; /** Which match algorithm is used by this matching setup */ diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java b/config-model/src/main/java/com/yahoo/schema/document/MatchType.java index dd4af0965d8..d2088e71282 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/MatchType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/MatchType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; public enum MatchType { TEXT("text"), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java b/config-model/src/main/java/com/yahoo/schema/document/Matching.java index ffac35ba61e..f70f31be0bd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Matching.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Matching.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.io.Serializable; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java b/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java index a1e2986b572..12880d0e1c8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/NormalizeLevel.java +++ b/config-model/src/main/java/com/yahoo/schema/document/NormalizeLevel.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; /** * class representing the character normalization diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java b/config-model/src/main/java/com/yahoo/schema/document/RankType.java index 6d84a9e97fc..067c1e7f266 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/RankType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/RankType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; /** * The rank type of a field. For now this is just a container of a string name. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java index 235677cd05e..31fd9747e2d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Ranking.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Ranking.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.io.Serializable; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java index b87bdd8907e..d300bd08bfd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDDocumentType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; @@ -11,9 +11,9 @@ import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationTypeRegistry; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.FieldSets; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.FieldSets; +import com.yahoo.schema.Schema; import java.io.Serializable; import java.util.ArrayList; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java b/config-model/src/main/java/com/yahoo/schema/document/SDField.java index 382c0d0ceb5..668b6388620 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/SDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/SDField.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; @@ -14,10 +14,10 @@ import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.fieldoperation.FieldOperation; -import com.yahoo.searchdefinition.fieldoperation.FieldOperationContainer; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.fieldoperation.FieldOperation; +import com.yahoo.schema.fieldoperation.FieldOperationContainer; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java b/config-model/src/main/java/com/yahoo/schema/document/Sorting.java index ef175f68ab8..2d0c9a5d27b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Sorting.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Sorting.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.io.Serializable; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java b/config-model/src/main/java/com/yahoo/schema/document/Stemming.java index 5b4862d4643..5ec844e2540 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/Stemming.java +++ b/config-model/src/main/java/com/yahoo/schema/document/Stemming.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.language.process.StemMode; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedField.java b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java index fef5e25c402..efc0674586d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedField.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; /** * A field that is imported from a field in a referenced document type and given an alias name. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java index b4d76445507..7ad4feb6d32 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporaryImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporaryImportedFields.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import java.util.Collections; import java.util.HashMap; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java index 1452ff430f3..3194a14a143 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDDocumentType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDDocumentType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataTypeName; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java index 8c17b607f94..e455fa78455 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TemporarySDField.java +++ b/config-model/src/main/java/com/yahoo/schema/document/TemporarySDField.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java index 767d6cc8009..8de8c7b64fd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/TypedKey.java +++ b/config-model/src/main/java/com/yahoo/schema/document/TypedKey.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import com.yahoo.document.DataType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java b/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java index b7ba2333b3c..3dc46a91c1b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/SDAnnotationType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/annotation/SDAnnotationType.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document.annotation; +package com.yahoo.schema.document.annotation; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; import com.yahoo.document.annotation.AnnotationType; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java b/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java index 717e778adb9..de9bd977823 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/document/annotation/TemporaryAnnotationReferenceDataType.java +++ b/config-model/src/main/java/com/yahoo/schema/document/annotation/TemporaryAnnotationReferenceDataType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document.annotation; +package com.yahoo.schema.document.annotation; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ConstantTensorTransformer.java index f6ed5abaa7f..4e320594918 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ConstantTensorTransformer.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ConstantTensorTransformer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; -import com.yahoo.searchdefinition.FeatureNames; +import com.yahoo.schema.FeatureNames; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; import com.yahoo.searchlib.rankingexpression.evaluation.Value; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java index 1334a7a3aad..86aedd4332a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/ExpressionTransforms.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/ExpressionTransforms.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.google.common.collect.ImmutableList; import com.yahoo.searchlib.rankingexpression.RankingExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionInliner.java index 340eec278e3..382d51747bb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionInliner.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionInliner.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.schema.RankProfile; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionShadower.java index 1f0112194cf..702e4ea220e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/FunctionShadower.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/FunctionShadower.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.schema.RankProfile; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/LightGBMFeatureConverter.java index 1bccac237d5..af5fa5ebeab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/LightGBMFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/LightGBMFeatureConverter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.rule.Arguments; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxFeatureConverter.java index 6944a1f9dd1..2277491cd47 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxFeatureConverter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.rule.Arguments; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java index 35ee9ddb9ed..4c38c257602 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/OnnxModelTransformer.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/OnnxModelTransformer.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.ImmutableSchema; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.rule.Arguments; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; @@ -53,15 +52,14 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans return transformFeature(feature, context.rankProfile()); } - public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile rankProfile) { - ImmutableSchema search = rankProfile.schema(); - final String featureName = feature.getName(); + public static ExpressionNode transformFeature(ReferenceNode feature, RankProfile profile) { + String featureName = feature.getName(); if ( ! featureName.equals("onnxModel") && ! featureName.equals("onnx")) return feature; Arguments arguments = feature.getArguments(); if (arguments.isEmpty()) throw new IllegalArgumentException("An " + featureName + " feature must take an argument referring to a " + - "onnx-model config or an ONNX file."); + "onnx-model config or an ONNX file."); if (arguments.expressions().size() > 3) throw new IllegalArgumentException("An " + featureName + " feature can have at most 3 arguments."); @@ -71,11 +69,11 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans // ONNX file that was transformed to Vespa ranking expressions. We then assume it is in the model store. String modelConfigName = getModelConfigName(feature.reference()); - OnnxModel onnxModel = search.onnxModels().get(modelConfigName); + OnnxModel onnxModel = profile.onnxModels().get(modelConfigName); if (onnxModel == null) { String path = asString(arguments.expressions().get(0)); ModelName modelName = new ModelName(null, Path.fromString(path), true); - ConvertedModel convertedModel = ConvertedModel.fromStore(search.applicationPackage(), modelName, path, rankProfile); + ConvertedModel convertedModel = ConvertedModel.fromStore(profile.schema().applicationPackage(), modelName, path, profile); FeatureArguments featureArguments = new FeatureArguments(arguments); return convertedModel.expression(featureArguments, null); } @@ -84,7 +82,7 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans String output = getModelOutput(feature.reference(), defaultOutput); if (! onnxModel.getOutputMap().containsValue(output)) { throw new IllegalArgumentException(featureName + " argument '" + output + - "' output not found in model '" + onnxModel.getFileName() + "'"); + "' output not found in model '" + onnxModel.getFileName() + "'"); } return new ReferenceNode("onnxModel", List.of(new ReferenceNode(modelConfigName)), output); } @@ -95,7 +93,7 @@ public class OnnxModelTransformer extends ExpressionTransformer<RankProfileTrans if (expr instanceof ReferenceNode) { // refers to onnx-model config return expr.toString(); } - if (expr instanceof ConstantNode) { // refers to an file path + if (expr instanceof ConstantNode) { // refers to a file path return asValidIdentifier(expr); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java index 9a732a63519..cfc859345ad 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/RankProfileTransformContext.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/RankProfileTransformContext.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.schema.RankProfile; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; @@ -33,7 +33,7 @@ public class RankProfileTransformContext extends TransformContext { QueryProfileRegistry queryProfiles, Map<Reference, TensorType> featureTypes, ImportedMlModels importedModels, - Map<String, RankProfile.Constant> constants, + Map<Reference, RankProfile.Constant> constants, Map<String, RankProfile.RankingExpressionFunction> inlineFunctions) { super(valuesOf(constants), rankProfile.typeContext(queryProfiles, featureTypes)); this.rankProfile = rankProfile; @@ -48,7 +48,7 @@ public class RankProfileTransformContext extends TransformContext { public Map<String, RankProfile.RankingExpressionFunction> inlineFunctions() { return inlineFunctions; } public Map<String, String> rankProperties() { return rankProperties; } - private static Map<String, Value> valuesOf(Map<String, RankProfile.Constant> constants) { + private static Map<String, Value> valuesOf(Map<Reference, RankProfile.Constant> constants) { return constants.values().stream() .filter(constant -> constant.value().isPresent()) .collect(Collectors.toMap(constant -> constant.name().simpleArgument().get(), diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java index 7b165d94cae..fd19c97036e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TensorFlowFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TensorFlowFeatureConverter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.rule.Arguments; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TokenTransformer.java index dbb32d88ef6..30d9a3766b3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/TokenTransformer.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/TokenTransformer.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.DoubleValue; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/XgboostFeatureConverter.java b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/XgboostFeatureConverter.java index 296ffd5f7fe..b05f9ba9166 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/expressiontransforms/XgboostFeatureConverter.java +++ b/config-model/src/main/java/com/yahoo/schema/expressiontransforms/XgboostFeatureConverter.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.expressiontransforms; +package com.yahoo.schema.expressiontransforms; import com.yahoo.path.Path; import com.yahoo.searchlib.rankingexpression.rule.Arguments; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AliasOperation.java index 1eb27cd31c2..b5648dde4fc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AliasOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AliasOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AttributeOperation.java index e528b1a9971..3983137129d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/AttributeOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/AttributeOperation.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; import com.yahoo.tensor.TensorType; import java.util.Locale; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/BoldingOperation.java index 63486a7656b..74e69e047ef 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/BoldingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/BoldingOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/DictionaryOperation.java index 5b2e0fb004f..a9a2ce7cbb1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/DictionaryOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/DictionaryOperation.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.SDField; /** * Represents operations controlling setup of dictionary used for queries diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java index 2778a147a18..126f594c371 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; /** * An operation on a field. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperationContainer.java index f81b1e42449..d5e52fadffa 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/FieldOperationContainer.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/FieldOperationContainer.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IdOperation.java index 2d9d2d0b8db..5e62742085f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IdOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IdOperation.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexOperation.java index 4611f11f023..ab5ffa25f33 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexOperation.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; - -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Index.Type; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +package com.yahoo.schema.fieldoperation; + +import com.yahoo.schema.Index; +import com.yahoo.schema.Index.Type; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import java.util.LinkedList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java index cdd3cc386a4..bb79a45831e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/IndexingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/IndexingOperation.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.parser.SimpleCharStream; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.parser.SimpleCharStream; import com.yahoo.vespa.indexinglanguage.ScriptParserContext; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.indexinglanguage.expressions.StatementExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/MatchOperation.java index 322f26bc0e3..a568b5b0f66 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/MatchOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/MatchOperation.java @@ -1,11 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.MatchAlgorithm; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.MatchAlgorithm; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/NormalizingOperation.java index cca7693119e..561c5b87899 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/NormalizingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/NormalizingOperation.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/QueryCommandOperation.java index fc2dd303746..d0e9feb41e4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/QueryCommandOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/QueryCommandOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankOperation.java index df9095ffcc5..bbc6208ba1b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankTypeOperation.java index 970e5bc0678..4a43a907549 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/RankTypeOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/RankTypeOperation.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Index; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SortingOperation.java index 148919c6de3..2e981a893ce 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SortingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SortingOperation.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Sorting; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StemmingOperation.java index 6ee137ff38c..a4bb00b0d07 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StemmingOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StemmingOperation.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StructFieldOperation.java index 285e1d97242..ac80f5023fc 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/StructFieldOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/StructFieldOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import java.util.Collections; import java.util.LinkedList; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldLongOperation.java index f776f7b1cec..4576b7a34fe 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldLongOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldLongOperation.java @@ -1,13 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; -import java.util.ArrayList; import java.util.Iterator; -import java.util.List; import java.util.Set; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldOperation.java index 5bd99fb3c8f..dd06d920aac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldOperation.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldShortOperation.java index e5b96940ac7..ccc22719f25 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryInFieldShortOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryInFieldShortOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryToOperation.java index 1378389c3e6..2d9cf3acf4e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/SummaryToOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/SummaryToOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightOperation.java index cc3a0ba687b..57c28d9bdb5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightOperation.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; /** * @author Einar M R Rosenvinge diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightedSetOperation.java index 11d731f814d..8fb0cc9fcdb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/fieldoperation/WeightedSetOperation.java +++ b/config-model/src/main/java/com/yahoo/schema/fieldoperation/WeightedSetOperation.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.fieldoperation; +package com.yahoo.schema.fieldoperation; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.document.Attribute; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import com.yahoo.document.WeightedSetDataType; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java index 0e79356abd2..fa656b72530 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedFields.java @@ -1,21 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; -import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Sorting; +import com.yahoo.schema.document.annotation.SDAnnotationType; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java index b04bfca3f76..bd628779b24 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedRanking.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedRanking.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; import java.util.List; @@ -32,7 +30,7 @@ public class ConvertParsedRanking { if (name.equals("default")) { return rankProfileRegistry.get(schema, "default"); } - return new RankProfile(name, schema, rankProfileRegistry, schema.rankingConstants()); + return new RankProfile(name, schema, rankProfileRegistry); } void convertRankProfile(Schema schema, ParsedRankProfile parsed) { @@ -42,8 +40,11 @@ public class ConvertParsedRanking { parsed.isStrict().ifPresent(value -> profile.setStrict(value)); - for (var constant : parsed.getConstants().entrySet()) - profile.addConstant(constant.getKey(), constant.getValue()); + for (var constant : parsed.getConstants().values()) + profile.add(constant); + + for (var onnxModel : parsed.getOnnxModels()) + profile.add(onnxModel); for (var input : parsed.getInputs().entrySet()) profile.addInput(input.getKey(), input.getValue()); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java index 9c2d29282ae..f3289621ce1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedSchemas.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedSchemas.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -11,15 +11,15 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.searchdefinition.DefaultRankProfile; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.UnrankedRankProfile; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; -import com.yahoo.searchdefinition.parser.ConvertParsedTypes.TypeResolver; +import com.yahoo.schema.DefaultRankProfile; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.UnrankedRankProfile; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; +import com.yahoo.schema.parser.ConvertParsedTypes.TypeResolver; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; @@ -28,7 +28,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.logging.Level; /** * Class converting a collection of schemas from the intermediate format. @@ -80,7 +79,7 @@ public class ConvertParsedSchemas { this.typeConverter = new ConvertParsedTypes(orderedInput, docMan); } - private Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap(); + private final Map<String, SDDocumentType> convertedDocuments = new LinkedHashMap<>(); public List<Schema> convertToSchemas() { typeConverter.convert(false); @@ -208,14 +207,12 @@ public class ConvertParsedSchemas { if (documentsOnly) { return; // skip ranking-only content, not used for document type generation } - for (var rankingConstant : parsed.getRankingConstants()) { - schema.rankingConstants().add(rankingConstant); - } - for (var onnxModel : parsed.getOnnxModels()) { - schema.onnxModels().add(onnxModel); - } - rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); - rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry, schema.rankingConstants())); + for (var constant : parsed.getConstants()) + schema.add(constant); + for (var onnxModel : parsed.getOnnxModels()) + schema.add(onnxModel); + rankProfileRegistry.add(new DefaultRankProfile(schema, rankProfileRegistry)); + rankProfileRegistry.add(new UnrankedRankProfile(schema, rankProfileRegistry)); var rankConverter = new ConvertParsedRanking(rankProfileRegistry); for (var rankProfile : parsed.getRankProfiles()) { rankConverter.convertRankProfile(schema, rankProfile); diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java index 7311c8a4216..9f1203ffc9f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertParsedTypes.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertParsedTypes.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; @@ -10,7 +10,7 @@ import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.OwnedStructDataType; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.schema.document.annotation.SDAnnotationType; import java.util.ArrayList; import java.util.Collection; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java index 8f25a586aa6..5509d11885c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ConvertSchemaCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ConvertSchemaCollection.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -10,8 +10,8 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import java.util.ArrayList; import java.util.List; @@ -118,7 +118,7 @@ public class ConvertSchemaCollection { } private void resolveStructInheritance() { - List<ParsedStruct> all = new ArrayList(); + List<ParsedStruct> all = new ArrayList<>(); for (var schema : orderedInput) { var doc = schema.getDocument(); for (var struct : doc.getStructs()) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/DictionaryOption.java b/config-model/src/main/java/com/yahoo/schema/parser/DictionaryOption.java index 095add74dd2..3acb51ace3f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/DictionaryOption.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/DictionaryOption.java @@ -1,4 +1,4 @@ -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; public enum DictionaryOption { HASH, BTREE, CASED, UNCASED diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java b/config-model/src/main/java/com/yahoo/schema/parser/InheritanceResolver.java index dbdcd4057e9..ad9acf2f095 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/InheritanceResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/InheritanceResolver.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.HashMap; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java index cdcd13619e8..8bb9bca3249 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/IntermediateCollection.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/IntermediateCollection.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; @@ -11,10 +11,8 @@ import com.yahoo.io.reader.NamedReader; import com.yahoo.yolean.Exceptions; import java.io.File; -import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; -import java.util.List; import java.util.Map; /** @@ -46,7 +44,7 @@ public class IntermediateCollection { public ParsedSchema addSchemaFromString(String input) throws ParseException { var stream = new SimpleCharStream(input); - var parser = new IntermediateParser(stream, deployLogger, modelProperties); + var parser = new SchemaParser(stream, deployLogger, modelProperties); try { var schema = parser.schema(); if (parsedSchemas.containsKey(schema.name())) { @@ -132,7 +130,7 @@ public class IntermediateCollection { throw new IllegalArgumentException("No schema named: " + schemaName); } var stream = new SimpleCharStream(IOUtils.readAll(reader.getReader())); - var parser = new IntermediateParser(stream, deployLogger, modelProperties); + var parser = new SchemaParser(stream, deployLogger, modelProperties); try { parser.rankProfile(schema); } catch (ParseException pe) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAnnotation.java index 3baac504135..c36656838f7 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAnnotation.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAnnotation.java @@ -1,4 +1,4 @@ -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java index b48bad89114..be8d20fbe93 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedAttribute.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedAttribute.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.LinkedHashMap; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedBlock.java index 151da352f2f..c20abf52bf3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedBlock.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedBlock.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; /** * Common methods for various Parsed* classes. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocument.java index ea138808289..281e7989885 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocument.java @@ -1,12 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; -import java.util.Collection; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Optional; /** * This class holds the extracted information after parsing a diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java index 25adc6f134f..93469a86fe3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedDocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedDocumentSummary.java @@ -1,5 +1,5 @@ -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedField.java index fcbb62b1229..a4df2ac6dc2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedField.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedField.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.document.Stemming; import java.util.ArrayList; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedFieldSet.java index 9103ed46631..9e8906a41a4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedFieldSet.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedFieldSet.java @@ -1,4 +1,4 @@ -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndex.java index a3504cd2bf7..cf70168e8d2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndex.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndex.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.Stemming; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndexingOp.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndexingOp.java index 4178de904de..3a2df2aac4c 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedIndexingOp.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedIndexingOp.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java index 9b51521ad2b..4d3c45ad67f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedMatchSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedMatchSettings.java @@ -1,8 +1,8 @@ -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.MatchAlgorithm; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.MatchAlgorithm; import java.util.Optional; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankFunction.java index a8ee9a24e69..73f1316d468 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankFunction.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankFunction.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java index 67f3672b2ac..64dd8dd0ad4 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedRankProfile.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedRankProfile.java @@ -1,14 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; -import com.yahoo.searchdefinition.RankProfile.MutateOperation; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile.MutateOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; -import com.yahoo.tensor.TensorType; import java.util.ArrayList; import java.util.Collections; @@ -52,8 +50,9 @@ class ParsedRankProfile extends ParsedBlock { private final Map<String, ParsedRankFunction> functions = new LinkedHashMap<>(); private final Map<String, String> fieldsRankType = new LinkedHashMap<>(); private final Map<String, List<String>> rankProperties = new LinkedHashMap<>(); - private final Map<String, RankProfile.Constant> constants = new LinkedHashMap<>(); + private final Map<Reference, RankProfile.Constant> constants = new LinkedHashMap<>(); private final Map<Reference, RankProfile.Input> inputs = new LinkedHashMap<>(); + private final List<OnnxModel> onnxModels = new ArrayList<>(); ParsedRankProfile(String name) { super(name, "rank-profile"); @@ -83,8 +82,9 @@ class ParsedRankProfile extends ParsedBlock { Map<String, Integer> getFieldsWithRankWeight() { return Collections.unmodifiableMap(fieldsRankWeight); } Map<String, String> getFieldsWithRankType() { return Collections.unmodifiableMap(fieldsRankType); } Map<String, List<String>> getRankProperties() { return Collections.unmodifiableMap(rankProperties); } - Map<String, RankProfile.Constant> getConstants() { return Collections.unmodifiableMap(constants); } + Map<Reference, RankProfile.Constant> getConstants() { return Collections.unmodifiableMap(constants); } Map<Reference, RankProfile.Input> getInputs() { return Collections.unmodifiableMap(inputs); } + List<OnnxModel> getOnnxModels() { return List.copyOf(onnxModels); } Optional<String> getInheritedSummaryFeatures() { return Optional.ofNullable(this.inheritedSummaryFeatures); } Optional<String> getSecondPhaseExpression() { return Optional.ofNullable(this.secondPhaseExpression); } @@ -101,9 +101,9 @@ class ParsedRankProfile extends ParsedBlock { this.inheritedSummaryFeatures = other; } - void addConstant(String name, RankProfile.Constant value) { - verifyThat(! constants.containsKey(name), "already has constant", name); - constants.put(name, value); + void add(RankProfile.Constant constant) { + verifyThat(! constants.containsKey(constant.name()), "already has constant", constant.name()); + constants.put(constant.name(), constant); } void addInput(Reference name, RankProfile.Input input) { @@ -111,6 +111,10 @@ class ParsedRankProfile extends ParsedBlock { inputs.put(name, input); } + void add(OnnxModel model) { + onnxModels.add(model); + } + void addFieldRankFilter(String field, boolean filter) { fieldsRankFilter.put(field, filter); } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSchema.java index 599dd6e2a7a..5ee483db044 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSchema.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSchema.java @@ -1,9 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.document.Stemming; +import com.yahoo.searchlib.rankingexpression.Reference; import java.util.ArrayList; import java.util.LinkedHashMap; @@ -14,9 +15,12 @@ import java.util.Optional; /** * This class holds the extracted information after parsing * one schema (.sd) file, using simple data structures - * as far as possible. Do not put advanced logic here! + * as far as possible. + * + * Do not put complicated logic here! + * * @author arnej27959 - **/ + */ public class ParsedSchema extends ParsedBlock { public static class ImportedField { @@ -36,11 +40,11 @@ public class ParsedSchema extends ParsedBlock { private Stemming defaultStemming = null; private final List<ImportedField> importedFields = new ArrayList<>(); private final List<OnnxModel> onnxModels = new ArrayList<>(); - private final List<RankingConstant> rankingConstants = new ArrayList<>(); + private final Map<Reference, RankProfile.Constant> constants = new LinkedHashMap<>(); private final List<String> inherited = new ArrayList<>(); private final List<String> inheritedByDocument = new ArrayList<>(); - private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap(); - private final Map<String, ParsedSchema> allResolvedInherits = new LinkedHashMap(); + private final Map<String, ParsedSchema> resolvedInherits = new LinkedHashMap<>(); + private final Map<String, ParsedSchema> allResolvedInherits = new LinkedHashMap<>(); private final Map<String, ParsedAnnotation> extraAnnotations = new LinkedHashMap<>(); private final Map<String, ParsedDocumentSummary> docSums = new LinkedHashMap<>(); private final Map<String, ParsedField> extraFields = new LinkedHashMap<>(); @@ -67,12 +71,12 @@ public class ParsedSchema extends ParsedBlock { List<ParsedFieldSet> getFieldSets() { return List.copyOf(fieldSets.values()); } List<ParsedIndex> getIndexes() { return List.copyOf(extraIndexes.values()); } List<ParsedStruct> getStructs() { return List.copyOf(extraStructs.values()); } - List<RankingConstant> getRankingConstants() { return List.copyOf(rankingConstants); } List<String> getInherited() { return List.copyOf(inherited); } List<String> getInheritedByDocument() { return List.copyOf(inheritedByDocument); } List<ParsedRankProfile> getRankProfiles() { return List.copyOf(rankProfiles.values()); } List<ParsedSchema> getResolvedInherits() { return List.copyOf(resolvedInherits.values()); } List<ParsedSchema> getAllResolvedInherits() { return List.copyOf(allResolvedInherits.values()); } + List<RankProfile.Constant> getConstants() { return List.copyOf(constants.values()); } void addAnnotation(ParsedAnnotation annotation) { String annName = annotation.name(); @@ -119,7 +123,7 @@ public class ParsedSchema extends ParsedBlock { extraIndexes.put(idxName, index); } - void addOnnxModel(OnnxModel model) { + void add(OnnxModel model) { onnxModels.add(model); } @@ -129,8 +133,8 @@ public class ParsedSchema extends ParsedBlock { rankProfiles.put(rpName, profile); } - void addRankingConstant(RankingConstant constant) { - rankingConstants.add(constant); + void add(RankProfile.Constant constant) { + constants.put(constant.name(), constant); } void addStruct(ParsedStruct struct) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSorting.java index 32c822bf25b..af84bbbb5bd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSorting.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSorting.java @@ -1,8 +1,8 @@ -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.searchdefinition.document.Sorting.Function; -import com.yahoo.searchdefinition.document.Sorting.Strength; +import com.yahoo.schema.document.Sorting.Function; +import com.yahoo.schema.document.Sorting.Strength; import java.util.Optional; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java index 376fd592c9d..abe14b3689f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedStruct.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedStruct.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.LinkedHashMap; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSummaryField.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java index 40e95045afd..38ee52c9d06 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedSummaryField.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedSummaryField.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import java.util.ArrayList; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java b/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java index bcf8d8c9172..9c3206a333a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/ParsedType.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/ParsedType.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.tensor.TensorType; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java b/config-model/src/main/java/com/yahoo/schema/parser/SimpleCharStream.java index 0b275c6a722..0a53e0477ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/SimpleCharStream.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/SimpleCharStream.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.javacc.FastCharStream; /** * @author Simon Thoresen Hult */ -public class SimpleCharStream extends FastCharStream implements com.yahoo.searchdefinition.parser.CharStream, +public class SimpleCharStream extends FastCharStream implements com.yahoo.schema.parser.CharStream, com.yahoo.vespa.indexinglanguage.parser.CharStream { public SimpleCharStream(String input) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java b/config-model/src/main/java/com/yahoo/schema/parser/Utils.java index e732bdad19c..cdb299c92df 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/parser/Utils.java +++ b/config-model/src/main/java/com/yahoo/schema/parser/Utils.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; /** * @author bjorncs diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java index 47ded54dcb6..d96cd88f6be 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFields.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableImportedComplexSDField; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableImportedComplexSDField; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java index 0be48d1fd25..ca81301da73 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AddExtraFieldsToDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AddExtraFieldsToDocument.java @@ -1,21 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.document.DataType; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.Field; -import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** - * This processor creates a {@link com.yahoo.searchdefinition.document.SDDocumentType} for each {@link Schema} + * This processor creates a {@link com.yahoo.schema.document.SDDocumentType} for each {@link Schema} * object which holds all the data that search * associates with a document described in a search definition file. This includes all extra fields, summary fields and * implicit fields. All non-indexed and non-summary fields are discarded. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java b/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java index 254b3743f52..6c2d62f37cb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AdjustPositionSummaryFields.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryField.Source; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributeProperties.java index 42e14a2c1ee..6c7dbaecbfb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributeProperties.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributeProperties.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java index 3c10ccb92dc..415f23f2786 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/AttributesImplicitWord.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/AttributesImplicitWord.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.MatchType; import com.yahoo.document.NumericDataType; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java b/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java index 575a037be91..53a3d462d54 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Bolding.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Bolding.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java index 6676e20c341..bdb1eed4b10 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BoolAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/BoolAttributeValidator.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java b/config-model/src/main/java/com/yahoo/schema/processing/BuiltInFieldSets.java index ce93c1a5c3f..514cbf225fd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/BuiltInFieldSets.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/BuiltInFieldSets.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DocumentType; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java index d7882c7f8fb..5bb5079fab6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/CreatePositionZCurve.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/CreatePositionZCurve.java @@ -1,17 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java index dc620e38e96..3209fd1703d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DictionaryProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DictionaryProcessor.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.NumericDataType; import com.yahoo.document.PrimitiveDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypes.java index 6ffe00458a8..a5b4ca9a71f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypes.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypes.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; import com.yahoo.document.PrimitiveDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.document.WeightedSetDataType; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java index 3759fc453df..0400292c7e5 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/DiversitySettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/DiversitySettingsValidator.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java index ca473a2029f..aa2d8293cac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ExactMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ExactMatch.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.indexinglanguage.ExpressionSearcher; import com.yahoo.vespa.indexinglanguage.expressions.ExactExpression; import com.yahoo.vespa.indexinglanguage.expressions.Expression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/FastAccessValidator.java index 60cc5c1cbb4..224000e6b64 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FastAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/FastAccessValidator.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.stream.Collectors; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java b/config-model/src/main/java/com/yahoo/schema/processing/FieldSetSettings.java index 1c50f78b539..f0c59ece1bf 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FieldSetSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/FieldSetSettings.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java b/config-model/src/main/java/com/yahoo/schema/processing/FilterFieldNames.java index 3f97bf83565..28973c82d42 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/FilterFieldNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/FilterFieldNames.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.RankProfile; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Iterator; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java index 0db6f4f05ba..4080e37003f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaries.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaries.java @@ -1,20 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import java.util.logging.Level; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** * Makes implicitly defined summaries into explicit summaries diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaryFields.java index df103dcb096..b17efbfe8e8 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImplicitSummaryFields.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java index e836caac10d..ee465be44f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ImportedFieldsResolver.java @@ -1,30 +1,30 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedComplexField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.ImportedSimpleField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.TemporaryImportedField; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.LinkedHashMap; import java.util.Map; import java.util.Optional; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isArrayOfSimpleStruct; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfPrimitiveType; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isMapOfSimpleStruct; /** * Iterates all imported fields from schema parsing and validates and resolves them into concrete fields from referenced document types. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java index 76e97ff0f2e..27101c47c7a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexFieldNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexFieldNames.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.container.search.QueryProfiles; /** @@ -37,7 +37,7 @@ public class IndexFieldNames extends Processor { * In {@link CreatePositionZCurve} we add some .position and .distance fields for pos fields. Make an exception for those for now. * TODO Vespa 8: Rename to _position and _distance and delete this method. * - * @param field an {@link com.yahoo.searchdefinition.document.SDField} + * @param field an {@link com.yahoo.schema.document.SDField} * @return true if allowed */ private boolean legalDottedPositionField(SDField field) { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java index e589bd8e91a..88e84d5289f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingInputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingInputs.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.Expression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java index b10d839535d..ea65a223686 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingOutputs.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingOutputs.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java index 242f5dab308..d8c1fb3125f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValidation.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValidation.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.MapDataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValues.java index 815e5a42df4..fa4b7d2bc40 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IndexingValues.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IndexingValues.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.InputExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java index ff6224d8d19..1d8480a8e99 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/IntegerIndex2Attribute.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/IntegerIndex2Attribute.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.NumericDataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java b/config-model/src/main/java/com/yahoo/schema/processing/LiteralBoost.java index afe004fd5f3..a84f895100a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/LiteralBoost.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/LiteralBoost.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.RankProfile; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Iterator; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeAliases.java index 14d75c6438e..7093242d0ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeAliases.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeAliases.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Index; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Index; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.ArrayList; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java index 49a56bafe2a..ea24bf0569d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MakeDefaultSummaryTheSuperSet.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MakeDefaultSummaryTheSuperSet.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchConsistency.java index 6cb0a86b481..5fb59e53ba9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchConsistency.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.IndexExpression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java index 4eae6b47833..7c1c255097f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchPhaseSettingsValidator.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/MatchedElementsOnlyResolver.java index 299f28ca573..ed95f87d7d6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MatchedElementsOnlyResolver.java @@ -1,21 +1,21 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; import com.yahoo.document.DataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ComplexAttributeFieldUtils; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isSupportedComplexField; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isSupportedComplexField; /** * Iterates all summary fields with 'matched-elements-only' and adjusts transform (if all struct-fields are attributes) diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java b/config-model/src/main/java/com/yahoo/schema/processing/MultifieldIndexHarmonizer.java index 39220b8cf8f..3a889085871 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MultifieldIndexHarmonizer.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MultifieldIndexHarmonizer.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.multifieldresolver.IndexCommandResolver; -import com.yahoo.searchdefinition.processing.multifieldresolver.RankTypeResolver; -import com.yahoo.searchdefinition.processing.multifieldresolver.StemmingResolver; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.multifieldresolver.IndexCommandResolver; +import com.yahoo.schema.processing.multifieldresolver.RankTypeResolver; +import com.yahoo.schema.processing.multifieldresolver.StemmingResolver; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java b/config-model/src/main/java/com/yahoo/schema/processing/MutableAttributes.java index 198f89caddb..854f6b2dddb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/MutableAttributes.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/MutableAttributes.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; public class MutableAttributes extends Processor { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java index 2ba149925e9..f1ff910be43 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/NGramMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/NGramMatch.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.indexinglanguage.expressions.*; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelConfigGenerator.java b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelConfigGenerator.java index fdbde08d926..ce56a4320d3 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelConfigGenerator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelConfigGenerator.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.expressiontransforms.OnnxModelTransformer; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.expressiontransforms.OnnxModelTransformer; import com.yahoo.searchlib.rankingexpression.rule.CompositeNode; import com.yahoo.searchlib.rankingexpression.rule.ConstantNode; import com.yahoo.searchlib.rankingexpression.rule.ExpressionNode; @@ -23,7 +23,7 @@ import java.util.Map; * * onnx("files/model.onnx", "path/to/output:1") * - * And generates an "onnx-model" configuration as if it was defined in the schema: + * And generates an "onnx-model" configuration as if it was defined in the profile: * * onnx-model files_model_onnx { * file: "files/model.onnx" @@ -45,31 +45,31 @@ public class OnnxModelConfigGenerator extends Processor { if (documentsOnly) return; for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { if (profile.getFirstPhaseRanking() != null) { - process(profile.getFirstPhaseRanking().getRoot()); + process(profile.getFirstPhaseRanking().getRoot(), profile); } if (profile.getSecondPhaseRanking() != null) { - process(profile.getSecondPhaseRanking().getRoot()); + process(profile.getSecondPhaseRanking().getRoot(), profile); } for (Map.Entry<String, RankProfile.RankingExpressionFunction> function : profile.getFunctions().entrySet()) { - process(function.getValue().function().getBody().getRoot()); + process(function.getValue().function().getBody().getRoot(), profile); } for (ReferenceNode feature : profile.getSummaryFeatures()) { - process(feature); + process(feature, profile); } } } - private void process(ExpressionNode node) { + private void process(ExpressionNode node, RankProfile profile) { if (node instanceof ReferenceNode) { - process((ReferenceNode)node); + process((ReferenceNode)node, profile); } else if (node instanceof CompositeNode) { for (ExpressionNode child : ((CompositeNode) node).children()) { - process(child); + process(child, profile); } } } - private void process(ReferenceNode feature) { + private void process(ReferenceNode feature, RankProfile profile) { if (feature.getName().equals("onnxModel") || feature.getName().equals("onnx")) { if (feature.getArguments().size() > 0) { if (feature.getArguments().expressions().get(0) instanceof ConstantNode) { @@ -85,11 +85,9 @@ public class OnnxModelConfigGenerator extends Processor { } } - OnnxModel onnxModel = schema.onnxModels().get(modelConfigName); - if (onnxModel == null) { - onnxModel = new OnnxModel(modelConfigName, path); - schema.onnxModels().add(onnxModel); - } + OnnxModel onnxModel = profile.onnxModels().get(modelConfigName); + if (onnxModel == null) + profile.add(new OnnxModel(modelConfigName, path)); } } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelTypeResolver.java index 4153cca4b5b..32229ea635b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OnnxModelTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/OnnxModelTypeResolver.java @@ -1,11 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.ml.OnnxModelInfo; @@ -28,9 +29,11 @@ public class OnnxModelTypeResolver extends Processor { @Override public void process(boolean validate, boolean documentsOnly) { if (documentsOnly) return; - for (OnnxModel onnxModel : schema.onnxModels().asMap().values()) { - OnnxModelInfo onnxModelInfo = OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage()); - onnxModel.setModelInfo(onnxModelInfo); + for (OnnxModel onnxModel : schema.declaredOnnxModels().values()) + onnxModel.setModelInfo(OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage())); + for (RankProfile profile : rankProfileRegistry.rankProfilesOf(schema)) { + for (OnnxModel onnxModel : profile.declaredOnnxModels().values()) + onnxModel.setModelInfo(OnnxModelInfo.load(onnxModel.getFileName(), schema.applicationPackage())); } } diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java index 3175862e900..a3b026fb724 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/OptimizeIlscript.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/OptimizeIlscript.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.indexinglanguage.ExpressionOptimizer; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java index 2ca4abae2c4..34bb6e1db2e 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PagedAttributeValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PagedAttributeValidator.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java index d7ae4c33fe1..280eae3d88b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/PredicateProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/PredicateProcessor.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.DataType; import com.yahoo.document.datatypes.IntegerFieldValue; import com.yahoo.document.datatypes.LongFieldValue; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java index c0ee0b5df53..63eca2121c1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processing.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processing.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.processing.multifieldresolver.RankProfileTypeSettingsProcessor; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.processing.multifieldresolver.RankProfileTypeSettingsProcessor; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.TestProperties; @@ -113,7 +113,7 @@ public class Processing { * * @param schema the search to process * @param deployLogger the log to log messages and warnings for application deployment to - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param rankProfileRegistry a {@link com.yahoo.schema.RankProfileRegistry} * @param queryProfiles the query profiles contained in the application this search is part of * @param processorsToSkip a set of processor classes we should not invoke in this. Useful for testing. */ @@ -132,7 +132,7 @@ public class Processing { * Runs rank profiles processors only. * * @param deployLogger the log to log messages and warnings for application deployment to - * @param rankProfileRegistry a {@link com.yahoo.searchdefinition.RankProfileRegistry} + * @param rankProfileRegistry a {@link com.yahoo.schema.RankProfileRegistry} * @param queryProfiles the query profiles contained in the application this search is part of */ public void processRankProfiles(DeployLogger deployLogger, RankProfileRegistry rankProfileRegistry, diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java index 590681ffb86..9768f33c27d 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/Processor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/Processor.java @@ -1,17 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.Index; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Iterator; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java index fdf1dc187cf..07f79f16334 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/RankingExpressionTypeResolver.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.MapEvaluationTypeContext; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.MapEvaluationTypeContext; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java index 09fdf9c65f6..43e39b1e546 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReferenceFieldsProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReferenceFieldsProcessor.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java b/config-model/src/main/java/com/yahoo/schema/processing/ReservedDocumentNames.java index 4a39a52a005..7eaf690d899 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedDocumentNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReservedDocumentNames.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.HashSet; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java index f4f920d9ec8..1ec4d5b58f2 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ReservedFunctionNames.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ReservedFunctionNames.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.google.common.collect.ImmutableSet; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.searchlib.rankingexpression.parser.RankingExpressionParserConstants; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java b/config-model/src/main/java/com/yahoo/schema/processing/SearchMustHaveDocument.java index 68717f1f06b..b90a5fdec98 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SearchMustHaveDocument.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SearchMustHaveDocument.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java b/config-model/src/main/java/com/yahoo/schema/processing/SetRankTypeEmptyOnFilters.java index 6631258813a..f84d6f19145 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SetRankTypeEmptyOnFilters.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SetRankTypeEmptyOnFilters.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java b/config-model/src/main/java/com/yahoo/schema/processing/SortingSettings.java index 33e9634323b..e0dfbab9780 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SortingSettings.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SortingSettings.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Sorting; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Sorting; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java b/config-model/src/main/java/com/yahoo/schema/processing/StringSettingsOnNonStringFields.java index 8c6b07515aa..8ca0b595907 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/StringSettingsOnNonStringFields.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/StringSettingsOnNonStringFields.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.CollectionDataType; import com.yahoo.document.NumericDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; public class StringSettingsOnNonStringFields extends Processor { diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java index e2e0cf94bb8..4fb45c3c68f 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryConsistency.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryConsistency.java @@ -1,20 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** * Ensure that summary field transforms for fields having the same name diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDiskAccessValidator.java index 0741b0fd6f1..40c38a350b0 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDiskAccessValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDiskAccessValidator.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SummaryClass; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; @@ -14,7 +14,7 @@ import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Optional; import java.util.logging.Level; -import static com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; +import static com.yahoo.schema.document.ComplexAttributeFieldUtils.isComplexFieldWithOnlyStructFieldAttributes; /** * Emits a warning for summaries which accesses disk. diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java index c350c6800ed..ed1f47611eb 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryDynamicStructsArrays.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryDynamicStructsArrays.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.*; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSource.java index c6f5b35aaa8..c8f201e2915 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSource.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSource.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SummaryClass; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SummaryClass; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java b/config-model/src/main/java/com/yahoo/schema/processing/SummaryNamesFieldCollisions.java index 2d0afe88281..da5dfeb407b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/SummaryNamesFieldCollisions.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/SummaryNamesFieldCollisions.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import java.util.HashMap; import java.util.Map; import com.yahoo.collections.Pair; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryField.Source; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java b/config-model/src/main/java/com/yahoo/schema/processing/TagType.java index 357575660ea..f511d572bc6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TagType.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TagType.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.*; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java index a253076f320..e0ce9917179 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TensorFieldProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TensorFieldProcessor.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.CollectionDataType; import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java index c2b41edf454..1783a3c7c63 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TextMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TextMatch.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.ExpressionVisitor; import com.yahoo.vespa.indexinglanguage.expressions.Expression; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java index d6fdb901ab7..1836cd631ad 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/TypedTransformProvider.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/TypedTransformProvider.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.indexinglanguage.ValueTransformProvider; import com.yahoo.vespa.indexinglanguage.expressions.*; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java index 7397f9a289c..a4773a42ed6 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UriHack.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/UriHack.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.Arrays; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java b/config-model/src/main/java/com/yahoo/schema/processing/UrlFieldValidator.java index 72f903f8365..63d4a342c72 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/UrlFieldValidator.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/UrlFieldValidator.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java index 68bf2511a4b..2327cf4d9c9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypes.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypes.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypesDocumentsOnly.java index e8b218a80ad..08771b40fe9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesDocumentsOnly.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldTypesDocumentsOnly.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.container.search.QueryProfiles; import java.util.HashMap; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldWithIndexSettingsCreatesIndex.java index 268a9f27b3c..5423defa74a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateFieldWithIndexSettingsCreatesIndex.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateFieldWithIndexSettingsCreatesIndex.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.Ranking; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.Ranking; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateStructTypeInheritance.java b/config-model/src/main/java/com/yahoo/schema/processing/ValidateStructTypeInheritance.java index d99832e3df6..cad555a24b1 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/ValidateStructTypeInheritance.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/ValidateStructTypeInheritance.java @@ -1,21 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; +import com.yahoo.schema.RankProfileRegistry; import com.yahoo.vespa.model.container.search.QueryProfiles; -import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.document.StructDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; import java.util.ArrayList; -import java.util.HashMap; -import java.util.Map; import java.util.HashSet; -import java.util.Set; /** * @author arnej diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java b/config-model/src/main/java/com/yahoo/schema/processing/WordMatch.java index 2a1afd616bc..1e312b71afd 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/WordMatch.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/WordMatch.java @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.container.search.QueryProfiles; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java index 56926436545..565a377f2a9 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/IndexCommandResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/IndexCommandResolver.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; +package com.yahoo.schema.processing.multifieldresolver; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; import java.util.ArrayList; import java.util.Arrays; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java index 80bb4faeaa5..ed8ad61706b 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/MultiFieldResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/MultiFieldResolver.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; +package com.yahoo.schema.processing.multifieldresolver; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; import java.util.List; /** diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java index fb7e67f2aab..3d79ac7d68a 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankProfileTypeSettingsProcessor.java @@ -1,20 +1,20 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; +package com.yahoo.schema.processing.multifieldresolver; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.TensorFieldType; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.processing.Processor; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.processing.Processor; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.model.container.search.QueryProfiles; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java index 231a97f30ec..6424fd8ba06 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/RankTypeResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/RankTypeResolver.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; +package com.yahoo.schema.processing.multifieldresolver; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.Schema; import java.util.List; import java.util.logging.Level; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java index 1bcf646c8ec..95d9a50a6ab 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/multifieldresolver/StemmingResolver.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/multifieldresolver/StemmingResolver.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing.multifieldresolver; +package com.yahoo.schema.processing.multifieldresolver; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; import java.util.List; import java.util.logging.Level; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java b/config-model/src/main/java/com/yahoo/schema/processing/package-info.java index 9d259154373..e81d50897ac 100644 --- a/config-model/src/main/java/com/yahoo/searchdefinition/processing/package-info.java +++ b/config-model/src/main/java/com/yahoo/schema/processing/package-info.java @@ -11,4 +11,4 @@ * New processors must be added to the list in Processing. */ @com.yahoo.api.annotations.PackageMarker -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java b/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java deleted file mode 100644 index c9c12100552..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/OnnxModels.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.FileRegistry; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; - -/** - * ONNX models tied to a search definition or global. - * - * @author lesters - */ -public class OnnxModels { - - private final FileRegistry fileRegistry; - - /** The schema this belongs to, or empty if it is global */ - private final Optional<Schema> owner; - - private final Map<String, OnnxModel> models = new HashMap<>(); - - public OnnxModels(FileRegistry fileRegistry, Optional<Schema> owner) { - this.fileRegistry = fileRegistry; - this.owner = owner; - } - - public void add(OnnxModel model) { - model.validate(); - model.register(fileRegistry); - String name = model.getName(); - models.put(name, model); - } - - public void add(Map<String, OnnxModel> models) { - models.values().forEach(this::add); - } - - public OnnxModel get(String name) { - var model = models.get(name); - if (model != null) return model; - if (owner.isPresent() && owner.get().inherited().isPresent()) - return owner.get().inherited().get().onnxModels().get(name); - return null; - } - - public boolean has(String name) { - boolean has = models.containsKey(name); - if (has) return true; - if (owner.isPresent() && owner.get().inherited().isPresent()) - return owner.get().inherited().get().onnxModels().has(name); - return false; - } - - public Map<String, OnnxModel> asMap() { - // Shortcuts - if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(models); - if (models.isEmpty()) return owner.get().inherited().get().onnxModels().asMap(); - - var allModels = new HashMap<>(owner.get().inherited().get().onnxModels().asMap()); - allModels.putAll(models); - return Collections.unmodifiableMap(allModels); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java deleted file mode 100644 index 377e19eedf2..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstant.java +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.tensor.TensorType; - -/** - * A global ranking constant distributed using file distribution. - * Ranking constants must be sent to some services to be useful - this is done - * by calling the sentTo method during the prepare phase of building models. - * - * @author arnej - * @author bratseth - */ -public class RankingConstant extends DistributableResource { - private TensorType tensorType = null; - - public RankingConstant(String name) { - super(name); - } - - public RankingConstant(String name, TensorType type, String fileName) { - this(name, type, fileName, PathType.FILE); - } - public RankingConstant(String name, TensorType type, String fileName, PathType pathType) { - super(name, fileName, pathType); - this.tensorType = type; - validate(); - } - - public void setType(TensorType type) { - this.tensorType = type; - } - - public TensorType getTensorType() { return tensorType; } - public String getType() { return tensorType.toString(); } - - public void validate() { - super.validate(); - if (tensorType == null) - throw new IllegalArgumentException("Ranking constant '" + getName() + "' must have a type."); - if (tensorType.dimensions().stream().anyMatch(d -> d.isIndexed() && d.size().isEmpty())) - throw new IllegalArgumentException("Illegal type in field " + getName() + " type " + tensorType + - ": Dense tensor dimensions must have a size"); - } - - public String toString() { - StringBuilder b = new StringBuilder(super.toString()) - .append("' of type '").append(tensorType).append("'"); - return b.toString(); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java b/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java deleted file mode 100644 index 5020e9a061c..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/RankingConstants.java +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; - -import com.yahoo.config.application.api.FileRegistry; - -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; -import java.util.function.Function; - -/** - * Constant values for ranking/model execution tied to a search definition, or globally to an application - * package - * - * @author bratseth - */ -public class RankingConstants { - - private final FileRegistry fileRegistry; - - /** The schema this belongs to, or empty if it is global */ - private final Optional<Schema> owner; - - private final Map<String, RankingConstant> constants = new ConcurrentHashMap<>(); - - public RankingConstants(FileRegistry fileRegistry, Optional<Schema> owner) { - this.fileRegistry = fileRegistry; - this.owner = owner; - } - - public void add(RankingConstant constant) { - constant.validate(); - constant.register(fileRegistry); - String name = constant.getName(); - RankingConstant prev = constants.putIfAbsent(name, constant); - if ( prev != null ) - throw new IllegalArgumentException("Ranking constant '" + name + "' defined twice"); - } - - public void putIfAbsent(RankingConstant constant) { - constant.validate(); - constant.register(fileRegistry); - String name = constant.getName(); - constants.putIfAbsent(name, constant); - } - - public void computeIfAbsent(String name, Function<? super String, ? extends RankingConstant> createConstant) { - constants.computeIfAbsent(name, key -> { - RankingConstant constant = createConstant.apply(key); - constant.validate(); - constant.register(fileRegistry); - return constant; - }); - } - - /** Returns the ranking constant with the given name, or null if not present */ - public RankingConstant get(String name) { - var constant = constants.get(name); - if (constant != null) return constant; - if (owner.isPresent() && owner.get().inherited().isPresent()) - return owner.get().inherited().get().rankingConstants().get(name); - return null; - } - - /** Returns a read-only map of the ranking constants in this indexed by name */ - public Map<String, RankingConstant> asMap() { - // Shortcuts - if (owner.isEmpty() || owner.get().inherited().isEmpty()) return Collections.unmodifiableMap(constants); - if (constants.isEmpty()) return owner.get().inherited().get().rankingConstants().asMap(); - - var allConstants = new HashMap<>(owner.get().inherited().get().rankingConstants().asMap()); - allConstants.putAll(constants); - return Collections.unmodifiableMap(allConstants); - } - -} diff --git a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java b/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java deleted file mode 100644 index 4c9e65f16c8..00000000000 --- a/config-model/src/main/java/com/yahoo/searchdefinition/derived/RankProfileList.java +++ /dev/null @@ -1,193 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; - -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; -import com.yahoo.config.model.api.ModelContext; -import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.OnnxModels; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.RankingConstants; -import com.yahoo.vespa.config.search.RankProfilesConfig; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.config.search.core.OnnxModelsConfig; -import com.yahoo.vespa.config.search.core.RankingConstantsConfig; -import com.yahoo.vespa.config.search.core.RankingExpressionsConfig; - -import java.util.ArrayList; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; -import java.util.logging.Logger; - -/** - * The derived rank profiles of a schema - * - * @author bratseth - */ -public class RankProfileList extends Derived implements RankProfilesConfig.Producer { - - private static final Logger log = Logger.getLogger(RankProfileList.class.getName()); - - private final Map<String, RawRankProfile> rankProfiles = new java.util.LinkedHashMap<>(); - private final RankingConstants rankingConstants; - private final LargeRankExpressions largeRankExpressions; - private final OnnxModels onnxModels; - - public static final RankProfileList empty = new RankProfileList(); - - private RankProfileList() { - rankingConstants = new RankingConstants(null, Optional.empty()); - largeRankExpressions = new LargeRankExpressions(null); - onnxModels = new OnnxModels(null, Optional.empty()); - } - - /** - * Creates a rank profile - * - * @param schema the schema this is a rank profile from - * @param attributeFields the attribute fields to create a ranking for - */ - public RankProfileList(Schema schema, - RankingConstants rankingConstants, - LargeRankExpressions largeRankExpressions, - OnnxModels onnxModels, - AttributeFields attributeFields, - RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - ModelContext.Properties deployProperties, - ExecutorService executor) { - setName(schema == null ? "default" : schema.getName()); - this.rankingConstants = rankingConstants; - this.largeRankExpressions = largeRankExpressions; - this.onnxModels = onnxModels; // as ONNX models come from parsing rank expressions - deriveRankProfiles(rankProfileRegistry, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); - } - - private boolean areDependenciesReady(RankProfile rank, RankProfileRegistry registry) { - return rank.inheritedNames().isEmpty() || - rankProfiles.keySet().containsAll(rank.inheritedNames()) || - (rank.schema() != null && rank.inheritedNames().stream().allMatch(name -> registry.resolve(rank.schema().getDocument(), name) != null)); - } - - private void deriveRankProfiles(RankProfileRegistry rankProfileRegistry, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - Schema schema, - AttributeFields attributeFields, - ModelContext.Properties deployProperties, - ExecutorService executor) { - if (schema != null) { // profiles belonging to a search have a default profile - RawRankProfile rawRank = new RawRankProfile(rankProfileRegistry.get(schema, "default"), - largeRankExpressions, queryProfiles, importedModels, attributeFields, deployProperties); - rankProfiles.put(rawRank.getName(), rawRank); - } - - Map<String, RankProfile> remaining = new LinkedHashMap<>(); - rankProfileRegistry.rankProfilesOf(schema).forEach(rank -> remaining.put(rank.name(), rank)); - remaining.remove("default"); - while (!remaining.isEmpty()) { - List<RankProfile> ready = new ArrayList<>(); - remaining.forEach((name, rank) -> { - if (areDependenciesReady(rank, rankProfileRegistry)) ready.add(rank); - }); - processRankProfiles(ready, queryProfiles, importedModels, schema, attributeFields, deployProperties, executor); - ready.forEach(rank -> remaining.remove(rank.name())); - } - } - private void processRankProfiles(List<RankProfile> ready, - QueryProfileRegistry queryProfiles, - ImportedMlModels importedModels, - Schema schema, - AttributeFields attributeFields, - ModelContext.Properties deployProperties, - ExecutorService executor) { - Map<String, Future<RawRankProfile>> futureRawRankProfiles = new LinkedHashMap<>(); - for (RankProfile rank : ready) { - if (schema == null) { - onnxModels.add(rank.onnxModels()); - } - - futureRawRankProfiles.put(rank.name(), executor.submit(() -> new RawRankProfile(rank, largeRankExpressions, queryProfiles, importedModels, - attributeFields, deployProperties))); - } - try { - for (Future<RawRankProfile> rawFuture : futureRawRankProfiles.values()) { - RawRankProfile rawRank = rawFuture.get(); - rankProfiles.put(rawRank.getName(), rawRank); - } - } catch (InterruptedException | ExecutionException e) { - throw new IllegalStateException(e); - } - } - - public OnnxModels getOnnxModels() { - return onnxModels; - } - - public Map<String, RawRankProfile> getRankProfiles() { - return rankProfiles; - } - - /** Returns the raw rank profile with the given name, or null if it is not present */ - public RawRankProfile getRankProfile(String name) { - return rankProfiles.get(name); - } - - @Override - public String getDerivedName() { return "rank-profiles"; } - - @Override - public void getConfig(RankProfilesConfig.Builder builder) { - for (RawRankProfile rank : rankProfiles.values() ) { - rank.getConfig(builder); - } - } - - public void getConfig(RankingExpressionsConfig.Builder builder) { - largeRankExpressions.expressions().forEach((expr) -> builder.expression.add(new RankingExpressionsConfig.Expression.Builder().name(expr.getName()).fileref(expr.getFileReference()))); - } - - public void getConfig(RankingConstantsConfig.Builder builder) { - for (RankingConstant constant : rankingConstants.asMap().values()) { - if ("".equals(constant.getFileReference())) - log.warning("Illegal file reference " + constant); // Let tests pass ... we should find a better way - else - builder.constant(new RankingConstantsConfig.Constant.Builder() - .name(constant.getName()) - .fileref(constant.getFileReference()) - .type(constant.getType())); - } - } - - public void getConfig(OnnxModelsConfig.Builder builder) { - for (OnnxModel model : onnxModels.asMap().values()) { - if ("".equals(model.getFileReference())) - log.warning("Illegal file reference " + model); // Let tests pass ... we should find a better way - else { - OnnxModelsConfig.Model.Builder modelBuilder = new OnnxModelsConfig.Model.Builder(); - modelBuilder.dry_run_on_setup(true); - modelBuilder.name(model.getName()); - modelBuilder.fileref(model.getFileReference()); - model.getInputMap().forEach((name, source) -> modelBuilder.input(new OnnxModelsConfig.Model.Input.Builder().name(name).source(source))); - model.getOutputMap().forEach((name, as) -> modelBuilder.output(new OnnxModelsConfig.Model.Output.Builder().name(name).as(as))); - if (model.getStatelessExecutionMode().isPresent()) - modelBuilder.stateless_execution_mode(model.getStatelessExecutionMode().get()); - if (model.getStatelessInterOpThreads().isPresent()) - modelBuilder.stateless_interop_threads(model.getStatelessInterOpThreads().get()); - if (model.getStatelessIntraOpThreads().isPresent()) - modelBuilder.stateless_intraop_threads(model.getStatelessIntraOpThreads().get()); - - builder.model(modelBuilder); - } - } - } -} diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java index a63728c403d..df78118009d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DataTypeRecognizer.java @@ -13,13 +13,12 @@ import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; import com.yahoo.document.annotation.AnnotationType; import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.document.internal.GeoPosType; import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.annotation.SDAnnotationType; +import com.yahoo.schema.document.annotation.SDAnnotationType; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java index 986746bd80c..1974774f7b5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentManager.java @@ -2,26 +2,21 @@ package com.yahoo.vespa.configmodel.producers; import com.yahoo.document.config.DocumentmanagerConfig; -import static com.yahoo.document.config.DocumentmanagerConfig.*; import com.yahoo.document.*; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedTemporaryType; import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.schema.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Set; diff --git a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java index 771d7c17162..bb57cd189c9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java +++ b/config-model/src/main/java/com/yahoo/vespa/configmodel/producers/DocumentTypes.java @@ -5,13 +5,12 @@ import com.yahoo.document.*; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.annotation.AnnotationReferenceDataType; import com.yahoo.document.annotation.AnnotationType; -import com.yahoo.documentmodel.DataTypeCollection; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedTemporaryType; import com.yahoo.documentmodel.TemporaryUnknownType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.schema.document.FieldSet; import com.yahoo.vespa.documentmodel.DocumentModel; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java index 2edc845d6b7..337f5e11329 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/DocumentSummary.java @@ -2,11 +2,9 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; diff --git a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java index da3070af55f..92a75cb051b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java +++ b/config-model/src/main/java/com/yahoo/vespa/documentmodel/SummaryField.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.document.TypedKey; +import com.yahoo.schema.document.TypedKey; import java.io.Serializable; import java.util.*; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java index c671ad5bd17..a919c19d754 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/VespaModel.java @@ -11,7 +11,6 @@ import com.yahoo.config.FileReference; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.model.ApplicationConfigProducerRoot; @@ -30,16 +29,14 @@ import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.container.QrConfig; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.OnnxModels; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.RankingConstants; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RankProfileList; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RankProfileList; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.config.ConfigKey; import com.yahoo.vespa.config.ConfigPayloadBuilder; @@ -125,9 +122,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** The global rank profiles of this model */ private final RankProfileList rankProfileList; - /** The global ranking constants of this model */ - private final RankingConstants rankingConstants; - /** The validation overrides of this. This is never null. */ private final ValidationOverrides validationOverrides; @@ -173,24 +167,17 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri version = deployState.getVespaVersion(); wantedNodeVersion = deployState.getWantedNodeVespaVersion(); fileReferencesRepository = new FileReferencesRepository(deployState.getFileRegistry()); - rankingConstants = new RankingConstants(deployState.getFileRegistry(), Optional.empty()); validationOverrides = deployState.validationOverrides(); applicationPackage = deployState.getApplicationPackage(); provisioned = deployState.provisioned(); VespaModelBuilder builder = new VespaDomBuilder(); root = builder.getRoot(VespaModel.ROOT_CONFIGID, deployState, this); - createGlobalRankProfiles(deployState, rankingConstants, deployState.getFileRegistry()); + createGlobalRankProfiles(deployState); rankProfileList = new RankProfileList(null, // null search -> global - rankingConstants, new LargeRankExpressions(deployState.getFileRegistry()), - new OnnxModels(deployState.getFileRegistry(), Optional.empty()), AttributeFields.empty, - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getProperties(), - deployState.getExecutor()); + deployState); HostSystem hostSystem = root.hostSystem(); if (complete) { // create a completed, frozen model @@ -259,9 +246,6 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri /** Returns the application package owning this */ public ApplicationPackage applicationPackage() { return applicationPackage; } - /** Returns the global ranking constants of this */ - public RankingConstants rankingConstants() { return rankingConstants; } - /** Creates a mutable model with no services instantiated */ public static VespaModel createIncomplete(DeployState deployState) throws IOException, SAXException { return new VespaModel(new NullConfigModelRegistry(), deployState, false); @@ -286,7 +270,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri * Creates a rank profile not attached to any search definition, for each imported model in the application package, * and adds it to the given rank profile registry. */ - private void createGlobalRankProfiles(DeployState deployState, RankingConstants rankingConstants, FileRegistry fileRegistry) { + private void createGlobalRankProfiles(DeployState deployState) { var importedModels = deployState.getImportedModels().all(); DeployLogger deployLogger = deployState.getDeployLogger(); RankProfileRegistry rankProfileRegistry = deployState.rankProfileRegistry(); @@ -295,12 +279,12 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if ( ! importedModels.isEmpty()) { // models/ directory is available for (ImportedMlModel model : importedModels) { // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. - OnnxModels onnxModels = onnxModelInfoFromSource(model, fileRegistry); - RankProfile profile = new RankProfile(model.name(), applicationPackage, deployLogger, rankProfileRegistry, rankingConstants, onnxModels); + RankProfile profile = new RankProfile(model.name(), applicationPackage, deployLogger, rankProfileRegistry); + addOnnxModelInfoFromSource(model, profile); rankProfileRegistry.add(profile); futureModels.add(deployState.getExecutor().submit(() -> { ConvertedModel convertedModel = ConvertedModel.fromSource(applicationPackage, new ModelName(model.name()), - model.name(), profile, queryProfiles.getRegistry(), model); + model.name(), profile, queryProfiles.getRegistry(), model); convertedModel.expressions().values().forEach(f -> profile.addFunction(f, false)); return convertedModel; })); @@ -312,8 +296,8 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri String modelName = generatedModelDir.getPath().last(); if (modelName.contains(".")) continue; // Name space: Not a global profile // Due to automatic naming not guaranteeing unique names, there must be a 1-1 between OnnxModels and global RankProfiles. - OnnxModels onnxModels = onnxModelInfoFromStore(modelName, fileRegistry); - RankProfile profile = new RankProfile(modelName, applicationPackage, deployLogger, rankProfileRegistry, rankingConstants, onnxModels); + RankProfile profile = new RankProfile(modelName, applicationPackage, deployLogger, rankProfileRegistry); + addOnnxModelInfoFromStore(modelName, profile); rankProfileRegistry.add(profile); futureModels.add(deployState.getExecutor().submit(() -> { ConvertedModel convertedModel = ConvertedModel.fromStore(applicationPackage, new ModelName(modelName), modelName, profile); @@ -332,27 +316,23 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri new Processing().processRankProfiles(deployLogger, rankProfileRegistry, queryProfiles, true, false); } - private OnnxModels onnxModelInfoFromSource(ImportedMlModel model, FileRegistry fileRegistry) { - OnnxModels onnxModels = new OnnxModels(fileRegistry, Optional.empty()); - if (model.modelType().equals(ImportedMlModel.ModelType.ONNX)) { + private void addOnnxModelInfoFromSource(ImportedMlModel model, RankProfile profile) { + if (model.modelType() == ImportedMlModel.ModelType.ONNX) { String path = model.source(); String applicationPath = this.applicationPackage.getFileReference(Path.fromString("")).toString(); if (path.startsWith(applicationPath)) { path = path.substring(applicationPath.length() + 1); } - loadOnnxModelInfo(onnxModels, model.name(), path); + addOnnxModelInfo(model.name(), path, profile); } - return onnxModels; } - private OnnxModels onnxModelInfoFromStore(String modelName, FileRegistry fileRegistry) { + private void addOnnxModelInfoFromStore(String modelName, RankProfile profile) { String path = ApplicationPackage.MODELS_DIR.append(modelName + ".onnx").toString(); - OnnxModels onnxModels = new OnnxModels(fileRegistry, Optional.empty()); - loadOnnxModelInfo(onnxModels, modelName, path); - return onnxModels; + addOnnxModelInfo(modelName, path, profile); } - private void loadOnnxModelInfo(OnnxModels onnxModels, String name, String path) { + private void addOnnxModelInfo(String name, String path, RankProfile profile) { boolean modelExists = OnnxModelInfo.modelExists(path, this.applicationPackage); if ( ! modelExists) { path = ApplicationPackage.MODELS_DIR.append(path).toString(); @@ -363,7 +343,7 @@ public final class VespaModel extends AbstractConfigProducerRoot implements Seri if (onnxModelInfo.getModelPath() != null) { OnnxModel onnxModel = new OnnxModel(name, onnxModelInfo.getModelPath()); onnxModel.setModelInfo(onnxModelInfo); - onnxModels.add(onnxModel); + profile.add(onnxModel); } } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java index c2b0a358a54..b2df37c47b2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/Metric.java @@ -1,9 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.admin.monitoring; -import javax.annotation.concurrent.Immutable; -import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; @@ -13,7 +10,6 @@ import java.util.Map; * @author trygve * @author gjoranv */ -@Immutable public class Metric { public final String name; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java index d145ce6bd24..01bf846d4cb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ComplexAttributeFieldsValidator.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SchemaInfo; -import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; -import com.yahoo.searchdefinition.document.GeoPos; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.document.ComplexAttributeFieldUtils; +import com.yahoo.schema.document.GeoPos; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java new file mode 100644 index 00000000000..bc91c7cbad2 --- /dev/null +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/ConstantValidator.java @@ -0,0 +1,83 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.config.application.api.ApplicationFile; +import com.yahoo.config.application.api.ApplicationPackage; +import com.yahoo.config.model.application.provider.FilesApplicationPackage; +import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.path.Path; +import com.yahoo.schema.DistributableResource; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.Schema; +import com.yahoo.vespa.model.VespaModel; +import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; + +import java.io.FileNotFoundException; + +/** + * RankingConstantsValidator validates all constant tensors (ranking constants) bundled with an application package + * + * @author Vegard Sjonfjell + */ +public class ConstantValidator extends Validator { + + @Override + public void validate(VespaModel model, DeployState deployState) { + var exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); + for (Schema schema : deployState.getSchemas()) { + for (var constant : schema.declaredConstants().values()) + validate(constant, deployState.getApplicationPackage(), exceptionMessageCollector); + for (var profile : deployState.rankProfileRegistry().rankProfilesOf(schema)) { + for (var constant : profile.declaredConstants().values()) + validate(constant, deployState.getApplicationPackage(), exceptionMessageCollector); + } + } + + if (exceptionMessageCollector.exceptionsOccurred) + throw new IllegalArgumentException(exceptionMessageCollector.combinedMessage); + } + + private void validate(RankProfile.Constant constant, + ApplicationPackage applicationPackage, + ExceptionMessageCollector exceptionMessageCollector) { + try { + validate(constant, applicationPackage); + } catch (InvalidConstantTensorException | FileNotFoundException exception) { + exceptionMessageCollector.add(constant, exception); + } + } + + private void validate(RankProfile.Constant rankingConstant, ApplicationPackage application) throws FileNotFoundException { + // Only validate file backed constants: + if (rankingConstant.valuePath().isEmpty()) return; + if (rankingConstant.pathType().get() != DistributableResource.PathType.FILE) return; + + String constantFile = rankingConstant.valuePath().get(); + if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) + && constantFile.startsWith(FilesApplicationPackage.preprocessed)) { + constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); + } + + ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); + new ConstantTensorJsonValidator().validate(constantFile, + rankingConstant.type(), + tensorApplicationFile.createReader()); + } + + private static class ExceptionMessageCollector { + + String combinedMessage; + boolean exceptionsOccurred = false; + + ExceptionMessageCollector(String messagePrelude) { + this.combinedMessage = messagePrelude; + } + + public void add(RankProfile.Constant constant, Exception exception) { + exceptionsOccurred = true; + combinedMessage += "\n" + constant.name() + " " + constant.type() + ": file:" + constant.valuePath().get() + + ": " + exception.getMessage(); + } + } + +} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java index 9de9c34737f..03f3cdba6cf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/NoPrefixForIndexes.java @@ -2,11 +2,11 @@ package com.yahoo.vespa.model.application.validation; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.MatchAlgorithm; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.MatchAlgorithm; +import com.yahoo.schema.Index; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; import com.yahoo.vespa.model.search.DocumentDatabase; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java index b9b7f122d63..5c531d28a8e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankSetupValidator.java @@ -10,7 +10,7 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.io.IOUtils; import com.yahoo.log.InvalidLogFormatException; import com.yahoo.log.LogMessage; -import com.yahoo.searchdefinition.DistributableResource; +import com.yahoo.schema.DistributableResource; import com.yahoo.system.ProcessExecuter; import com.yahoo.text.StringUtilities; import com.yahoo.vespa.config.search.AttributesConfig; @@ -152,7 +152,7 @@ public class RankSetupValidator extends Validator { List<String> config = new ArrayList<>(); // Assist verify-ranksetup in finding the actual ONNX model files - writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().onnxModels().asMap().values()); + writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getRankProfileList().getOnnxModels().asMap().values()); writeExtraVerifyRankSetupConfig(config, db.getDerivedConfiguration().getSchema().rankExpressionFiles().expressions()); config.sort(String::compareTo); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java deleted file mode 100644 index 82c793af70a..00000000000 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidator.java +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.config.application.api.ApplicationFile; -import com.yahoo.config.application.api.ApplicationPackage; -import com.yahoo.config.model.application.provider.FilesApplicationPackage; -import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.path.Path; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.application.validation.ConstantTensorJsonValidator.InvalidConstantTensorException; - -import java.io.FileNotFoundException; - -/** - * RankingConstantsValidator validates all constant tensors (ranking constants) bundled with an application package - * - * @author Vegard Sjonfjell - */ -public class RankingConstantsValidator extends Validator { - - private static class ExceptionMessageCollector { - - String combinedMessage; - boolean exceptionsOccurred = false; - - ExceptionMessageCollector(String messagePrelude) { - this.combinedMessage = messagePrelude; - } - - public ExceptionMessageCollector add(Throwable throwable, String rcName, String rcFilename) { - exceptionsOccurred = true; - combinedMessage += String.format("\nRanking constant '%s' (%s): %s", rcName, rcFilename, throwable.getMessage()); - return this; - } - } - - static class TensorValidationException extends IllegalArgumentException { - TensorValidationException(String message) { - super(message); - } - } - - @Override - public void validate(VespaModel model, DeployState deployState) { - ApplicationPackage applicationPackage = deployState.getApplicationPackage(); - ExceptionMessageCollector exceptionMessageCollector = new ExceptionMessageCollector("Invalid constant tensor file(s):"); - - for (Schema schema : deployState.getSchemas()) { - for (RankingConstant rc : schema.rankingConstants().asMap().values()) { - try { - validateRankingConstant(rc, applicationPackage); - } catch (InvalidConstantTensorException | FileNotFoundException ex) { - exceptionMessageCollector.add(ex, rc.getName(), rc.getFileName()); - } - } - } - - if (exceptionMessageCollector.exceptionsOccurred) { - throw new TensorValidationException(exceptionMessageCollector.combinedMessage); - } - } - - private void validateRankingConstant(RankingConstant rankingConstant, ApplicationPackage application) throws FileNotFoundException { - // TODO: Handle validation of URI soon too. - if (rankingConstant.getPathType() == RankingConstant.PathType.FILE) { - String constantFile = rankingConstant.getFileName(); - if (application.getFileReference(Path.fromString("")).getAbsolutePath().endsWith(FilesApplicationPackage.preprocessed) && - constantFile.startsWith(FilesApplicationPackage.preprocessed)) { - constantFile = constantFile.substring(FilesApplicationPackage.preprocessed.length()); - } - - ApplicationFile tensorApplicationFile = application.getFile(Path.fromString(constantFile)); - new ConstantTensorJsonValidator().validate(constantFile, - rankingConstant.getTensorType(), - tensorApplicationFile.createReader()); - } - } - -} diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java index 162cbe7495f..f62247d788e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/SearchDataTypeValidator.java @@ -11,10 +11,10 @@ import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.document.TensorDataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SchemaInfo; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java index 438b28948c6..458f8dfbd68 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/StreamingValidator.java @@ -6,9 +6,9 @@ import com.yahoo.config.model.deploy.DeployState; import com.yahoo.document.DataType; import com.yahoo.document.NumericDataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.MatchType; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.MatchType; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.SearchCluster; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java index 4ba809a58ba..ce61d3edc3b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/Validation.java @@ -77,7 +77,7 @@ public class Validation { new NoPrefixForIndexes().validate(model, deployState); new DeploymentSpecValidator().validate(model, deployState); new ValidationOverridesValidator().validate(model, deployState); - new RankingConstantsValidator().validate(model, deployState); + new ConstantValidator().validate(model, deployState); new SecretStoreValidator().validate(model, deployState); new EndpointCertificateSecretsValidator().validate(model, deployState); new AccessControlFilterValidator().validate(model, deployState); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java index 2820d0984db..0563aa02149 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/StreamingSearchClusterChangeValidator.java @@ -6,8 +6,8 @@ import com.yahoo.config.model.api.ConfigChangeAction; import com.yahoo.config.model.api.ServiceInfo; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.Attribute; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Attribute; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.change.search.ChangeMessageBuilder; import com.yahoo.vespa.model.application.validation.change.search.DocumentTypeChangeValidator; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java index 2a3c5fdc6b0..cfb641882f6 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidator.java @@ -5,12 +5,12 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.IndexSchema; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.HnswIndexParams; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.IndexSchema; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.HnswIndexParams; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; @@ -110,6 +110,7 @@ public class AttributeChangeValidator { Attribute current = currentFields.getAttribute(next.getName()); if (current != null) { validateAttributePredicate(id, current, next, Attribute::isFastSearch, "fast-search", result); + validateAttributePredicate(id, current, next, Attribute::isFastRank, "fast-rank", result); validateAttributePredicate(id, current, next, Attribute::isFastAccess, "fast-access", result); validateAttributeProperty(id, current, next, AttributeChangeValidator::extractDictionaryType, "dictionary: btree/hash", result); validateAttributeProperty(id, current, next, AttributeChangeValidator::extractDictionaryCase, "dictionary: cased/uncased", result); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java index b461b38c75f..bbfa939f8a3 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeMessageBuilder.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.application.validation.change.search; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.NormalizeLevel; -import com.yahoo.searchdefinition.document.Stemming; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.Matching; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.NormalizeLevel; +import com.yahoo.schema.document.Stemming; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java index f6f6b6abdee..a6fcfd652a0 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/IndexingScriptChangeValidator.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.model.application.validation.change.search; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableSDField; import com.yahoo.vespa.indexinglanguage.ExpressionConverter; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.expressions.OutputExpression; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java index a10d2c36de1..0909afb71e9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/application/validation/change/search/StructFieldAttributeChangeValidator.java @@ -8,9 +8,9 @@ import com.yahoo.document.Field; import com.yahoo.document.MapDataType; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.ComplexAttributeFieldUtils; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.ComplexAttributeFieldUtils; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRestartAction; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java index 71d325adb7d..640bede6b62 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/NodesSpecification.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.ConfigModelContext; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; @@ -56,12 +57,16 @@ public class NodesSpecification { /** The ID of the cluster referencing this node specification, if any */ private final Optional<String> combinedId; + /** The cloud account to use for nodes in this spec, if any */ + private final Optional<CloudAccount> cloudAccount; + private NodesSpecification(ClusterResources min, ClusterResources max, boolean dedicated, Version version, boolean required, boolean canFail, boolean exclusive, Optional<DockerImage> dockerImageRepo, - Optional<String> combinedId) { + Optional<String> combinedId, + Optional<CloudAccount> cloudAccount) { if (max.smallerThan(min)) throw new IllegalArgumentException("Min resources must be larger or equal to max resources, but " + max + " is smaller than " + min); @@ -83,10 +88,12 @@ public class NodesSpecification { this.exclusive = exclusive; this.dockerImageRepo = dockerImageRepo; this.combinedId = combinedId; + this.cloudAccount = cloudAccount; } private static NodesSpecification create(boolean dedicated, boolean canFail, Version version, - ModelElement nodesElement, Optional<DockerImage> dockerImageRepo) { + ModelElement nodesElement, Optional<DockerImage> dockerImageRepo, + Optional<CloudAccount> cloudAccount) { var resolvedElement = resolveElement(nodesElement); var combinedId = findCombinedId(nodesElement, resolvedElement); var resources = toResources(resolvedElement); @@ -98,7 +105,8 @@ public class NodesSpecification { canFail, resolvedElement.booleanAttribute("exclusive", false), dockerImageToUse(resolvedElement, dockerImageRepo), - combinedId); + combinedId, + cloudAccount); } private static Pair<ClusterResources, ClusterResources> toResources(ModelElement nodesElement) { @@ -125,7 +133,8 @@ public class NodesSpecification { ! context.getDeployState().getProperties().isBootstrap(), context.getDeployState().getWantedNodeVespaVersion(), nodesElement, - context.getDeployState().getWantedDockerImageRepo()); + context.getDeployState().getWantedDockerImageRepo(), + context.getDeployState().getProperties().cloudAccount()); } /** @@ -154,7 +163,8 @@ public class NodesSpecification { ! context.getDeployState().getProperties().isBootstrap(), context.getDeployState().getWantedNodeVespaVersion(), nodesElement, - context.getDeployState().getWantedDockerImageRepo())); + context.getDeployState().getWantedDockerImageRepo(), + context.getDeployState().getProperties().cloudAccount())); } /** @@ -169,7 +179,8 @@ public class NodesSpecification { ! context.getDeployState().getProperties().isBootstrap(), false, context.getDeployState().getWantedDockerImageRepo(), - Optional.empty()); + Optional.empty(), + context.getDeployState().getProperties().cloudAccount()); } /** Returns a requirement from <code>count</code> dedicated nodes in one group */ @@ -182,7 +193,8 @@ public class NodesSpecification { ! context.getDeployState().getProperties().isBootstrap(), false, context.getDeployState().getWantedDockerImageRepo(), - Optional.empty()); + Optional.empty(), + context.getDeployState().getProperties().cloudAccount()); } /** @@ -206,7 +218,8 @@ public class NodesSpecification { ! context.getDeployState().getProperties().isBootstrap(), false, context.getDeployState().getWantedDockerImageRepo(), - Optional.empty()); + Optional.empty(), + context.getDeployState().getProperties().cloudAccount()); } public ClusterResources minResources() { return min; } @@ -239,7 +252,7 @@ public class NodesSpecification { .dockerImageRepository(dockerImageRepo) .stateful(stateful) .build(); - return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail), logger); + return hostSystem.allocateHosts(cluster, Capacity.from(min, max, required, canFail, cloudAccount), logger); } private static Pair<NodeResources, NodeResources> nodeResources(ModelElement nodesElement) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java index 40df899c94d..cb8e6ba85ff 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerModelEvaluation.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.model.container; import ai.vespa.models.evaluation.ModelsEvaluator; import com.yahoo.osgi.provider.model.ComponentModel; -import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.schema.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java index 07a48e94399..0c65aee798b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/component/ConnectionLogComponent.java @@ -10,21 +10,26 @@ public class ConnectionLogComponent extends SimpleComponent implements Connectio private final String logDirectoryName; private final String clusterName; + private final boolean isHostedVespa; public ConnectionLogComponent(ContainerCluster<?> cluster, Class<? extends ConnectionLog> cls, String logDirectoryName) { - this(cls, logDirectoryName, cluster.getName()); + this(cls, logDirectoryName, cluster.getName(), cluster.isHostedVespa()); } - public ConnectionLogComponent(Class<? extends ConnectionLog> cls, String logDirectoryName, String clusterName) { + public ConnectionLogComponent(Class<? extends ConnectionLog> cls, String logDirectoryName, String clusterName, boolean isHostedVespa) { super(cls.getName()); this.logDirectoryName = logDirectoryName; this.clusterName = clusterName; + this.isHostedVespa = isHostedVespa; } @Override public void getConfig(ConnectionLogConfig.Builder builder) { builder.cluster(clusterName); builder.logDirectoryName(logDirectoryName); + if (isHostedVespa) { + builder.useClusterIdInFileName(false); + } builder.queueSize(-1); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java index 6532fed9abf..4afde25cafd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTester.java @@ -18,7 +18,7 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.filedistribution.fileacquirer.FileAcquirer; import com.yahoo.filedistribution.fileacquirer.MockFileAcquirer; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.schema.derived.RankProfileList; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; import com.yahoo.vespa.config.search.core.RankingConstantsConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java index 708e7c75ac8..97c3f85dced 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/ContainerSearch.java @@ -8,7 +8,7 @@ import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.pagetemplates.PageTemplatesConfig; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfilesConfig; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; import com.yahoo.vespa.model.container.component.Component; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java index 1ea84a33d2d..3d409263d8a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/search/searchchain/LocalProvider.java @@ -7,7 +7,7 @@ import com.yahoo.component.chain.model.ChainSpecification; import com.yahoo.component.chain.model.ChainedComponentModel; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.prelude.cluster.QrMonitorConfig; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.search.config.ClusterConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java index 170e22b6164..0bd93c6d0df 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilder.java @@ -34,8 +34,8 @@ import com.yahoo.config.provision.Zone; import com.yahoo.container.logging.FileConnectionLog; import com.yahoo.osgi.provider.model.ComponentModel; import com.yahoo.search.rendering.RendererRegistry; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.derived.RankProfileList; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.derived.RankProfileList; import com.yahoo.security.X509CertificateUtils; import com.yahoo.text.XML; import com.yahoo.vespa.defaults.Defaults; @@ -422,8 +422,13 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { protected void addAccessLogs(DeployState deployState, ApplicationContainerCluster cluster, Element spec) { List<Element> accessLogElements = getAccessLogElements(spec); - for (Element accessLog : accessLogElements) { - AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); + if (cluster.isHostedVespa() && !accessLogElements.isEmpty()) { + log.logApplicationPackage( + Level.WARNING, "Applications are not allowed to override the 'accesslog' element"); + } else { + for (Element accessLog : accessLogElements) { + AccessLogBuilder.buildIfNotDisabled(deployState, cluster, accessLog).ifPresent(cluster::addComponent); + } } if (accessLogElements.isEmpty() && deployState.getAccessLoggingEnabledByDefault()) @@ -431,7 +436,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { // Add connection log if access log is configured if (cluster.getAllComponents().stream().anyMatch(component -> component instanceof AccessLogComponent)) { - cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "qrs")); + // TODO: clean up after Vespa 8 + if (cluster.isHostedVespa() || deployState.getVespaVersion().getMajor() == 8) { + cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "access")); + } else { + cluster.addComponent(new ConnectionLogComponent(cluster, FileConnectionLog.class, "qrs")); + } } } @@ -588,7 +598,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { Element onnxElement = XML.getChild(modelEvaluationElement, "onnx"); Element modelsElement = XML.getChild(onnxElement, "models"); for (Element modelElement : XML.getChildren(modelsElement, "model") ) { - OnnxModel onnxModel = profiles.getOnnxModels().get(modelElement.getAttribute("name")); + OnnxModel onnxModel = profiles.getOnnxModels().asMap().get(modelElement.getAttribute("name")); if (onnxModel == null) continue; // Skip if model is not found onnxModel.setStatelessExecutionMode(getStringValue(modelElement, "execution-mode", null)); @@ -1105,7 +1115,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private final Element nodesElement; private final DeployLogger logger; private final boolean legacyOptions; - private final boolean failDeploymentWithInvalidJvmOptions; private final boolean isHosted; public JvmOptions(ContainerCluster<?> cluster, Element nodesElement, DeployState deployState, boolean legacyOptions) { @@ -1113,7 +1122,6 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { this.nodesElement = nodesElement; this.logger = deployState.getDeployLogger(); this.legacyOptions = legacyOptions; - this.failDeploymentWithInvalidJvmOptions = deployState.featureFlags().failDeploymentWithInvalidJvmOptions(); this.isHosted = deployState.isHosted(); } @@ -1185,7 +1193,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String message = "Invalid or misplaced JVM options in services.xml: " + String.join(",", invalidOptions) + "." + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; - if (failDeploymentWithInvalidJvmOptions && isHosted) + if (isHosted) throw new IllegalArgumentException(message); else logger.logApplicationPackage(WARNING, message); @@ -1206,14 +1214,12 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { private final String jvmGcOptions; private final DeployLogger logger; private final boolean isHosted; - private final boolean failDeploymentWithInvalidJvmOptions; public JvmGcOptions(DeployState deployState, String jvmGcOptions) { this.deployState = deployState; this.jvmGcOptions = jvmGcOptions; this.logger = deployState.getDeployLogger(); this.isHosted = deployState.isHosted(); - this.failDeploymentWithInvalidJvmOptions = deployState.featureFlags().failDeploymentWithInvalidJvmOptions(); } private String build() { @@ -1251,7 +1257,7 @@ public class ContainerModelBuilder extends ConfigModelBuilder<ContainerModel> { String message = "Invalid or misplaced JVM GC options in services.xml: " + String.join(",", options) + "." + " See https://docs.vespa.ai/en/reference/services-container.html#jvm"; - if (failDeploymentWithInvalidJvmOptions && isHosted) + if (isHosted) throw new IllegalArgumentException(message); else logger.logApplicationPackage(WARNING, message); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 82321216519..4be5153293f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -5,8 +5,8 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.builder.xml.dom.DomSearchTuningBuilder; @@ -273,8 +273,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> Optional<Tuning> tuning = Optional.ofNullable(this.tuning); if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, - clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), - fractionOfMemoryReserved); + clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), + deployState.featureFlags().loadCodeAsHugePages(), fractionOfMemoryReserved); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); @@ -381,18 +381,11 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> hasAnyNonIndexedCluster = true; ddbB.inputdoctypename(type.getFullName().getName()) .configid(findStreamingCluster(docTypeName).get().getDocumentDBConfigId()) - .mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING) - .feeding.concurrency(0.0); + .mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING); } else if (hasIndexingModeIndexed(type)) { getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); - if (tuning != null && tuning.searchNode != null && tuning.searchNode.feeding != null) { - ddbB.feeding.concurrency(tuning.searchNode.feeding.concurrency); - } else { - ddbB.feeding.concurrency(defaultFeedConcurrency); - } } else { hasAnyNonIndexedCluster = true; - ddbB.feeding.concurrency(0.0); ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 37ba9cde2cd..776d23e5227 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -51,7 +51,6 @@ import com.yahoo.vespa.model.content.storagecluster.StorageCluster; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.search.Tuning; import org.w3c.dom.Element; - import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; @@ -62,8 +61,6 @@ import java.util.Set; import java.util.TreeMap; import java.util.logging.Level; -import static com.yahoo.config.provision.NodeResources.DiskSpeed; -import static com.yahoo.config.provision.NodeResources.StorageType; import static java.util.stream.Collectors.toList; /** @@ -328,17 +325,14 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce } } - public static final NodeResources clusterControllerResources = new NodeResources(0.25, 1.14, 10, 0.3, DiskSpeed.any, StorageType.any); - private ClusterControllerContainerCluster getDedicatedSharedControllers(ModelElement contentElement, Admin admin, ConfigModelContext context, DeployState deployState, String clusterName) { if (admin.getClusterControllers() == null) { - NodeResources nodeResources = clusterControllerResources.with(deployState.featureFlags().adminClusterArchitecture()); NodesSpecification spec = NodesSpecification.requiredFromSharedParents(deployState.zone().environment().isProduction() ? 3 : 1, - nodeResources, + NodeResources.unspecified(), contentElement, context); Collection<HostResource> hosts = spec.provision(admin.hostSystem(), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java index 216bd916b5d..9ba87fd24bf 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/ml/ConvertedModel.java @@ -11,10 +11,9 @@ import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; import com.yahoo.searchlib.rankingexpression.ExpressionFunction; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.Reference; @@ -265,12 +264,12 @@ public class ConvertedModel { private static Map<String, ExpressionFunction> convertStored(ModelStore store, RankProfile profile) { for (Pair<String, Tensor> constant : store.readSmallConstants()) { - profile.addConstant(constant.getFirst(), - new RankProfile.Constant(FeatureNames.asConstantFeature(constant.getFirst()), constant.getSecond())); + var name = FeatureNames.asConstantFeature(constant.getFirst()); + profile.add(new RankProfile.Constant(name, constant.getSecond())); } - for (RankingConstant constant : store.readLargeConstants()) { - profile.rankingConstants().putIfAbsent(constant); + for (RankProfile.Constant constant : store.readLargeConstants()) { + profile.add(constant); } for (Pair<String, RankingExpression> function : store.readFunctions()) { @@ -298,8 +297,8 @@ public class ConvertedModel { String constantValueString) { Tensor constantValue = Tensor.from(constantValueString); store.writeSmallConstant(constantName, constantValue); - profile.addConstant(constantName, - new RankProfile.Constant(FeatureNames.asConstantFeature(constantName), constantValue)); + Reference name = FeatureNames.asConstantFeature(constantName); + profile.add(new RankProfile.Constant(name, constantValue)); } private static void transformLargeConstant(ModelStore store, @@ -318,10 +317,11 @@ public class ConvertedModel { constantsReplacedByFunctions.add(constantName); // will replace constant(constantName) by constantName later } else { - profile.rankingConstants().computeIfAbsent(constantName, name -> { - Path constantPath = store.writeLargeConstant(name, constantValue); - return new RankingConstant(name, constantValue.type(), constantPath.toString()); - }); + var constantReference = FeatureNames.asConstantFeature(constantName); + if ( ! profile.constants().containsKey(constantReference)) { + Path constantPath = store.writeLargeConstant(constantName, constantValue); + profile.add(new RankProfile.Constant(constantReference, constantValue.type(), constantPath.toString())); + } } } @@ -548,15 +548,17 @@ public class ConvertedModel { } /** - * Reads the information about all the large (aka ranking) constants stored in the application package + * Reads the information about all the large constants stored in the application package * (the constant value itself is replicated with file distribution). */ - List<RankingConstant> readLargeConstants() { + List<RankProfile.Constant> readLargeConstants() { try { - List<RankingConstant> constants = new ArrayList<>(); + List<RankProfile.Constant> constants = new ArrayList<>(); for (ApplicationFile constantFile : application.getFile(modelFiles.largeConstantsInfoPath()).listFiles()) { String[] parts = IOUtils.readAll(constantFile.createReader()).split(":"); - constants.add(new RankingConstant(parts[0], TensorType.fromSpec(parts[1]), parts[2])); + constants.add(new RankProfile.Constant(FeatureNames.asConstantFeature(parts[0]), + TensorType.fromSpec(parts[1]), + parts[2])); } return constants; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java index e628907068d..64a1015d056 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/DocumentDatabase.java @@ -4,7 +4,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.ImportedFieldsConfig; import com.yahoo.vespa.config.search.IndexschemaConfig; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index d5bf12df559..d63ae451f39 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -7,9 +7,9 @@ import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.derived.SchemaInfo; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.SchemaInfo; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.config.search.DispatchConfig.DistributionPolicy; @@ -189,13 +189,7 @@ public class IndexedSearchCluster extends SearchCluster for (SchemaInfo spec : schemas().values()) { if (spec.fullSchema() instanceof DocumentOnlySchema) continue; DocumentDatabase db = new DocumentDatabase(this, spec.fullSchema().getName(), - new DerivedConfiguration(spec.fullSchema(), - deployState.getDeployLogger(), - deployState.getProperties(), - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getExecutor())); + new DerivedConfiguration(spec.fullSchema(), deployState)); documentDbs.add(db); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java index b4082308195..06a5929a430 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SchemaDefinitionXMLHandler.java @@ -1,11 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.model.builder.xml.dom.ModelElement; import java.io.Serializable; -import java.util.Collection; import java.util.List; /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 1141af6d79d..d727e6d6395 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -3,11 +3,8 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.search.config.SchemaInfoConfig; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.derived.SchemaInfo; -import com.yahoo.searchdefinition.derived.SummaryMap; -import com.yahoo.searchlib.rankingexpression.Reference; -import com.yahoo.tensor.TensorType; +import com.yahoo.schema.derived.SchemaInfo; +import com.yahoo.schema.derived.SummaryMap; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; @@ -15,7 +12,7 @@ import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.config.model.producer.AbstractConfigProducer; import java.util.Collections; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 6e3f3e1ebf5..e7c0968f99e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -95,8 +95,9 @@ public class SearchNode extends AbstractService implements @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { - return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved); + return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), + deployState.featureFlags().loadCodeAsHugePages(), fractionOfMemoryReserved); } } @@ -104,26 +105,22 @@ public class SearchNode extends AbstractService implements public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + boolean loadCodeAsHugePages, double fractionOfMemoryReserved) { return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, loadCodeAsHugePages, fractionOfMemoryReserved); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + boolean loadCodeAsHugePages, double fractionOfMemoryReserved) { + super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; - setPropertiesElastic(clusterName, distributionKey); - } - - private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, - boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { - super(parent, name); this.isHostedVespa = isHostedVespa; + if (loadCodeAsHugePages) { + addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", "true"); + } this.fractionOfMemoryReserved = fractionOfMemoryReserved; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -136,6 +133,7 @@ public class SearchNode extends AbstractService implements // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + setPropertiesElastic(clusterName, distributionKey); } private void setPropertiesElastic(String clusterName, int distributionKey) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index df266ab77dd..724ba70510f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -4,9 +4,9 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.SummaryConfig; @@ -77,13 +77,7 @@ public class StreamingSearchCluster extends SearchCluster implements if ( ! schema.getName().equals(docTypeName)) throw new IllegalArgumentException("Document type name '" + docTypeName + "' must be the same as the schema name '" + schema.getName() + "'"); - this.schemaConfig = new DerivedConfiguration(schema, - deployState.getDeployLogger(), - deployState.getProperties(), - deployState.rankProfileRegistry(), - deployState.getQueryProfiles().getRegistry(), - deployState.getImportedModels(), - deployState.getExecutor()); + this.schemaConfig = new DerivedConfiguration(schema, deployState); } @Override diff --git a/config-model/src/main/javacc/IntermediateParser.jj b/config-model/src/main/javacc/SchemaParser.jj index 240c89d3171..47b7909eb69 100644 --- a/config-model/src/main/javacc/IntermediateParser.jj +++ b/config-model/src/main/javacc/SchemaParser.jj @@ -9,9 +9,9 @@ options { USER_CHAR_STREAM = true; } -PARSER_BEGIN(IntermediateParser) +PARSER_BEGIN(SchemaParser) -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.api.ModelContext; @@ -19,22 +19,21 @@ import com.yahoo.language.Linguistics; import com.yahoo.language.process.Embedder; import com.yahoo.language.simple.SimpleLinguistics; import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.DistributableResource; -import com.yahoo.searchdefinition.OnnxModel; -import com.yahoo.searchdefinition.RankProfile.DiversitySettings; -import com.yahoo.searchdefinition.RankProfile.MatchPhaseSettings; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankingConstant; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.MatchAlgorithm; -import com.yahoo.searchdefinition.document.HnswIndexParams; -import com.yahoo.searchdefinition.document.Sorting; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.FeatureNames; -import com.yahoo.searchdefinition.fieldoperation.IndexingOperation; +import com.yahoo.schema.DistributableResource; +import com.yahoo.schema.OnnxModel; +import com.yahoo.schema.RankProfile.DiversitySettings; +import com.yahoo.schema.RankProfile.MatchPhaseSettings; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.MatchAlgorithm; +import com.yahoo.schema.document.HnswIndexParams; +import com.yahoo.schema.document.Sorting; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.fieldoperation.IndexingOperation; import com.yahoo.searchlib.rankingexpression.FeatureList; import com.yahoo.searchlib.rankingexpression.Reference; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; @@ -59,13 +58,13 @@ import java.util.logging.Level; * * @author bratseth */ -public class IntermediateParser { +public class SchemaParser { private DeployLogger deployLogger; private ModelContext.Properties properties; /** Creates a parser. */ - public IntermediateParser(SimpleCharStream stream, + public SchemaParser(SimpleCharStream stream, DeployLogger deployLogger, ModelContext.Properties properties) { @@ -123,7 +122,7 @@ public class IntermediateParser { } } -PARSER_END(IntermediateParser) +PARSER_END(SchemaParser) // -------------------------------------------------------------------------------- @@ -428,7 +427,7 @@ void rootSchemaItem(ParsedSchema schema) : { } | structOutside(schema) | annotationOutside(schema) | fieldSet(schema) - | onnxModel(schema) + | onnxModelInSchema(schema) // Deprecated: TODO: Emit warning when on Vespa 8 ) } @@ -1668,15 +1667,15 @@ void indexBody(ParsedIndex index) : double threshold; } { - ( <PREFIX> { index.setPrefix(true); } - | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); } - | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(Stemming.get(str)); } - | <ARITY> <COLON> arity = integer() { index.setArity(arity); } - | <LOWERBOUND> <COLON> num = consumeLong() { index.setLowerBound(num); } - | <UPPERBOUND> <COLON> num = consumeLong() { index.setUpperBound(num); } - | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = consumeFloat() { index.setDensePostingListThreshold(threshold); } - | <ENABLE_BM25> { index.setEnableBm25(true); } - | hnswIndex(index) { } + ( <PREFIX> { index.setPrefix(true); } + | <ALIAS> <COLON> str = identifierWithDash() { index.addAlias(str); } + | <STEMMING> <COLON> str = identifierWithDash() { index.setStemming(Stemming.get(str)); } + | <ARITY> <COLON> arity = integer() { index.setArity(arity); } + | <LOWERBOUND> <COLON> num = longValue() { index.setLowerBound(num); } + | <UPPERBOUND> <COLON> num = longValue() { index.setUpperBound(num); } + | <DENSEPOSTINGLISTTHRESHOLD> <COLON> threshold = floatValue() { index.setDensePostingListThreshold(threshold); } + | <ENABLE_BM25> { index.setEnableBm25(true); } + | hnswIndex(index) { } ) } @@ -1704,31 +1703,38 @@ void hnswIndexBody(HnswIndexParams.Builder params) : | <MULTITHREADEDINDEXING> <COLON> bool = bool() { params.setMultiThreadedIndexing(bool); } ) } -/** - * Consumes a onnx-model block of a schema element. - * - * @param schema the schema object to add content to. - */ -void onnxModel(ParsedSchema schema) : +void onnxModelInSchema(ParsedSchema schema) : +{ + OnnxModel onnxModel; +} +{ + onnxModel = onnxModel() { schema.add(onnxModel); } +} + +void onnxModelInProfile(ParsedRankProfile profile) : +{ + OnnxModel onnxModel; +} +{ + onnxModel = onnxModel() { profile.add(onnxModel); } +} + +/** Consumes an onnx-model block. */ +OnnxModel onnxModel() : { String name; OnnxModel onnxModel; } { - ( <ONNXMODEL> name = identifier() - { - onnxModel = new OnnxModel(name); - } + ( <ONNXMODEL> name = identifier() { onnxModel = new OnnxModel(name); } lbrace() (onnxModelItem(onnxModel) (<NL>)*)+ <RBRACE> ) - { - schema.addOnnxModel(onnxModel); - } + { return onnxModel; } } /** - * This rule consumes an onnx-model block. + * Consumes an onnx-model block. * - * @param onnxModel The onnxModel to modify. + * @param onnxModel the onnxModel to modify */ void onnxModelItem(OnnxModel onnxModel) : { @@ -1768,13 +1774,15 @@ void rankingConstant(ParsedSchema schema) : { ( <CONSTANT> name = identifier() lbrace() (path = fileItem() { pathType = DistributableResource.PathType.FILE; } - | path = uriItem() { pathType = DistributableResource.PathType.URI; } // Undocumented. TODO: Remove oin Vespa 8 + | path = uriItem() { pathType = DistributableResource.PathType.URI; } // Undocumented. TODO: Remove on Vespa 8 | type = tensorTypeWithPrefix(rankingConstantErrorMessage(name)) (<NL>)* )+ <RBRACE> ) { - schema.addRankingConstant(new RankingConstant(name, type, path, pathType)); + if (type == null) throw new IllegalArgumentException("constant '" + name + "' must have a type"); + if (path == null) throw new IllegalArgumentException("constant '" + name + "' must have a file"); + schema.add(new RankProfile.Constant(FeatureNames.asConstantFeature(name), type, path, pathType)); } } @@ -1785,6 +1793,7 @@ String fileItem() : { (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; } } + String uriItem() : { String path; @@ -1848,6 +1857,7 @@ void rankProfileItem(ParsedSchema schema, ParsedRankProfile profile) : { } | constants(schema, profile) | matchFeatures(profile) | summaryFeatures(profile) + | onnxModelInProfile(profile) | strict(profile) ) } @@ -1896,7 +1906,7 @@ String mutate_expr() : Number constant = null; } { - (("+=" | "-=" | "=") { op = token.image; } constant = consumeNumber()) + (("+=" | "-=" | "=") { op = token.image; } constant = number()) { return constant != null ? (op + constant) : op; } } @@ -1968,9 +1978,9 @@ void matchPhaseItem(MatchPhaseSettings settings) : | <ORDER> <COLON> ( <ASCENDING> { settings.setAscending(true); } | <DESCENDING> { settings.setAscending(false); } ) | <MAXHITS> <COLON> num = integer() { settings.setMaxHits(num); } - | <MAXFILTERCOVERAGE> <COLON> coverage = consumeFloat() { settings.setMaxFilterCoverage(coverage); } - | <EVALUATION_POINT> <COLON> multiplier = consumeFloat() { settings.setEvaluationPoint(multiplier); } - | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = consumeFloat() { settings.setPrePostFilterTippingPoint(multiplier); } + | <MAXFILTERCOVERAGE> <COLON> coverage = floatValue() { settings.setMaxFilterCoverage(coverage); } + | <EVALUATION_POINT> <COLON> multiplier = floatValue() { settings.setEvaluationPoint(multiplier); } + | <PRE_POST_FILTER_TIPPING_POINT> <COLON> multiplier = floatValue() { settings.setPrePostFilterTippingPoint(multiplier); } ) } @@ -1999,7 +2009,7 @@ void diversityItem(DiversitySettings settings) : { ( <ATTRIBUTE> <COLON> str = identifier() { settings.setAttribute(str); } | <MIN_GROUPS> <COLON> num = integer() { settings.setMinGroups(num); } - | <CUTOFF_FACTOR> <COLON> multiplier = consumeFloat() { settings.setCutoffFactor(multiplier); } + | <CUTOFF_FACTOR> <COLON> multiplier = floatValue() { settings.setCutoffFactor(multiplier); } | <CUTOFF_STRATEGY> <COLON> ( <STRICT> { settings.setCutoffStrategy(Diversity.CutoffStrategy.strict); } | <LOOSE> { settings.setCutoffStrategy(Diversity.CutoffStrategy.loose); } @@ -2029,9 +2039,9 @@ void firstPhaseItem(ParsedRankProfile profile) : double dropLimit; } { - ( expression = expression() { profile.setFirstPhaseRanking(expression); } - | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } - | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = consumeFloat()) { profile.setRankScoreDropLimit(dropLimit); } + ( expression = expression() { profile.setFirstPhaseRanking(expression); } + | (<KEEPRANKCOUNT> <COLON> keepRankCount = integer()) { profile.setKeepRankCount(keepRankCount); } + | (<RANKSCOREDROPLIMIT> <COLON> dropLimit = floatValue()) { profile.setRankScoreDropLimit(dropLimit); } ) } @@ -2080,10 +2090,24 @@ void input(ParsedRankProfile profile) : Tensor defaultValue = null; } { - reference = queryFeature() ( type = valueType(reference))? ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? + reference = inputName() ( type = valueType(reference))? ( <COLON> (<NL>)* defaultValue = tensorValue(type) )? { profile.addInput(reference, new RankProfile.Input(reference, type, Optional.ofNullable(defaultValue))); } } +/** Returns the reference "query(name)" for both "query(name)" and "name". */ +Reference inputName() : +{ + String name; +} +{ + ( + ( <QUERY> "(" name = identifier() ")" ) + | + name = identifier() + ) + { return FeatureNames.asQueryFeature(name); } +} + TensorType valueType(Reference reference) : { TensorType type; @@ -2098,19 +2122,10 @@ TensorType valueType(Reference reference) : { return type; } } -Reference queryFeature() : -{ - String argument; -} -{ - <QUERY> "(" argument = identifier() ")" - { return Reference.simple("query", argument); } -} - /** - * This rule consumes a summary-features block of a rank profile. + * Consumes a summary-features block of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void summaryFeatures(ParsedRankProfile profile) : { @@ -2143,9 +2158,9 @@ void strict(ParsedRankProfile profile) : } /** - * This rule consumes a match-features block of a rank profile. + * Consumes a match-features block of a rank profile. * - * @param profile The rank profile to modify. + * @param profile the rank profile to modify */ void matchFeatures(ParsedRankProfile profile) : { @@ -2240,7 +2255,7 @@ void termwiseLimit(ParsedRankProfile profile) : double num; } { - (<TERMWISELIMIT> <COLON> num = consumeFloat()) { profile.setTermwiseLimit(num); } + (<TERMWISELIMIT> <COLON> num = floatValue()) { profile.setTermwiseLimit(num); } } /** @@ -2253,7 +2268,7 @@ void postFilterThreshold(ParsedRankProfile profile) : double threshold; } { - (<POSTFILTERTHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setPostFilterThreshold(threshold); } + (<POSTFILTERTHRESHOLD> <COLON> threshold = floatValue()) { profile.setPostFilterThreshold(threshold); } } /** @@ -2266,7 +2281,7 @@ void approximateThreshold(ParsedRankProfile profile) : double threshold; } { - (<APPROXIMATETHRESHOLD> <COLON> threshold = consumeFloat()) { profile.setApproximateThreshold(threshold); } + (<APPROXIMATETHRESHOLD> <COLON> threshold = floatValue()) { profile.setApproximateThreshold(threshold); } } /** @@ -2367,7 +2382,7 @@ void rankDegradationBinSize() : double freq; } { - <RPBINSIZE> <COLON> freq = consumeFloat() + <RPBINSIZE> <COLON> freq = floatValue() { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'doc-frequency' in 'rank-degradation' is deprecated and has no effect."); } } @@ -2392,7 +2407,7 @@ void rankDegradationPosbinSize() : double avgOcc; } { - <RPPOSBINSIZE> <COLON> avgOcc = consumeFloat() + <RPPOSBINSIZE> <COLON> avgOcc = floatValue() { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'occurrences-per-doc' in 'rank-degradation' is deprecated and has no effect."); } } @@ -2415,7 +2430,7 @@ void rankDegradation() : double freq; } { - ( <RANKDEGRADATIONFREQ> <COLON> freq = consumeFloat() + ( <RANKDEGRADATIONFREQ> <COLON> freq = floatValue() { deployLogger.logApplicationPackage(Level.WARNING, "Specifying 'rank-degradation-frequency' in 'rank-profile' is deprecated and has no effect."); } | <RANKDEGRADATION> lbrace() ( rankDegradationItem() (<NL>)*)+ <RBRACE> ) @@ -2448,9 +2463,9 @@ void constant(ParsedSchema schema, ParsedRankProfile profile) : LOOKAHEAD(4) ( ( type = valueType(name) )? <COLON> (<NL>)* ( value = tensorValue(type) | valuePath = fileItem()) { if (value != null) - profile.addConstant(name.simpleArgument().get(), new RankProfile.Constant(name, value)); + profile.add(new RankProfile.Constant(name, value)); else - schema.addRankingConstant(new RankingConstant(name.simpleArgument().get(), type, valuePath, DistributableResource.PathType.FILE)); // TODO JON: Move to RankProfile + profile.add(new RankProfile.Constant(name, type, valuePath)); } ) | // Deprecated forms (TODO: Add warning on Vespa 8): @@ -2465,9 +2480,11 @@ Reference constantName() : String name; } { - ( <CONSTANT> "(" name = identifier() ")" ) - | - name = identifier() + ( + ( <CONSTANT> "(" name = identifier() ")" ) + | + name = identifier() + ) { return FeatureNames.asConstantFeature(name); } } @@ -2478,7 +2495,7 @@ void constantValue(ParsedRankProfile profile, Reference name) : } { <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) - { profile.addConstant(name.simpleArgument().get(), new RankProfile.Constant(name, Tensor.from(value.image))); } + { profile.add(new RankProfile.Constant(name, Tensor.from(value.image))); } } // Deprecated form @@ -2491,8 +2508,7 @@ void constantTensor(ParsedRankProfile profile, Reference name) : <LBRACE> (<NL>)* (( tensorString = tensorValuePrefixedByValue() | type = tensorTypeWithPrefix(constantTensorErrorMessage(profile.name(), name)) ) (<NL>)* )* <RBRACE> - { profile.addConstant(name.simpleArgument().get(), - new RankProfile.Constant(name, type != null ? Tensor.from(type, tensorString) : Tensor.from(tensorString))); + { profile.add(new RankProfile.Constant(name, type != null ? Tensor.from(type, tensorString) : Tensor.from(tensorString))); } } @@ -2511,7 +2527,7 @@ Tensor tensorValue(TensorType type) : Number doubleValue = null; } { - ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = consumeNumber() ) + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = number() ) { if (doubleValue != null) { if (type.rank() > 0) @@ -2525,7 +2541,10 @@ Tensor tensorValue(TensorType type) : /** A mapped or mixed tensor value. */ void mappedTensorValue(Tensor.Builder builder) : {} { - "{" ( mappedTensorBlock(builder) )* ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* "}" + "{" + ( mappedTensorBlock(builder) )* + ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* + "}" } @@ -2534,11 +2553,11 @@ void mappedTensorBlock(Tensor.Builder builder) : TensorAddress mappedAddress; } { - mappedAddress = tensorAddress(builder.type().mappedSubtype()) <COLON> (<NL>)* + mappedAddress = tensorAddress(builder.type()) <COLON> (<NL>)* ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) } -void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builder) : +void indexedTensorBlockValues(TensorAddress mappedAddress, Tensor.Builder builder) : { List<Double> values = new ArrayList<Double>(); } @@ -2550,7 +2569,7 @@ void indexedTensorBlockValues(TensorAddress sparseAddress, Tensor.Builder builde for (int i = 0; i < values.size(); i++ ) { arrayValues[i] = values.get(i); } - boundBuilder.block(sparseAddress, arrayValues); + boundBuilder.block(mappedAddress, arrayValues); } } @@ -2583,7 +2602,7 @@ void indexedTensorValue(List<Double> values) : Number value; } { - value = consumeNumber() + value = number() { values.add(value.doubleValue()); } } @@ -2598,7 +2617,7 @@ void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : TensorAddress tensorAddress(TensorType type) : { - TensorAddress.Builder builder = new TensorAddress.Builder(type); + TensorAddress.Builder builder = new TensorAddress.PartialBuilder(type); String label; } { @@ -2634,7 +2653,7 @@ double tensorCellValue() : Number value; } { - value = consumeNumber() + value = number() { return value.doubleValue(); } } @@ -2907,7 +2926,7 @@ int integer() : { } } /** Consumes a long or integer token and returns its numeric value. */ -long consumeLong() : { } +long longValue() : { } { ( <INTEGER> { return Long.parseLong(token.image); } | <LONG> { return Long.parseLong(token.image.substring(0, token.image.length()-1)); } @@ -2915,17 +2934,17 @@ long consumeLong() : { } } /** Consumes a floating-point token and returns its numeric value. */ -double consumeFloat() : { } +double floatValue() : { } { <DOUBLE> { return Double.valueOf(token.image); } } -Number consumeNumber() : +Number number() : { Number num; } { - (num = consumeFloat() | num = consumeLong()) { return num; } + ( num = floatValue() | num = longValue() ) { return num; } } /** Consumes an opening brace with leading and trailing newline tokens. */ diff --git a/config-model/src/main/resources/schema/deployment.rnc b/config-model/src/main/resources/schema/deployment.rnc index 08b23f77bc9..3abced8e04a 100644 --- a/config-model/src/main/resources/schema/deployment.rnc +++ b/config-model/src/main/resources/schema/deployment.rnc @@ -7,6 +7,7 @@ start = element deployment { attribute major-version { text }? & attribute athenz-domain { xsd:string }? & attribute athenz-service { xsd:string }? & + attribute cloud-account { xsd:string }? & Step } @@ -33,6 +34,7 @@ PrimitiveStep = Instance = element instance { attribute id { xsd:string } & attribute athenz-service { xsd:string }? & + attribute cloud-account { xsd:string }? & StepExceptInstance } @@ -110,6 +112,7 @@ ProdTest = element test { Region = element region { attribute active { xsd:boolean }? & attribute athenz-service { xsd:string }? & + attribute cloud-account { xsd:string }? & text } diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd index a6564d59e16..d06ca2677a1 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/neuralnet.sd @@ -75,11 +75,19 @@ search neuralnet { query(W_0) tensor(x[9],hidden[9]) query(b_0) tensor(hidden[9]) query(W_1) tensor(hidden[9],out[9]) - query(b_1) tensor(out[9]) + b_1 tensor(out[9]) query(W_out) tensor(out[9]) query(b_out) tensor(out[1]):[1.0] } + inputs { + query(foo): 5.5 + } + + rank-properties { + query(bar): 5.5 + } + } rank-profile defaultRankProfile inherits default { diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg index cd3abcb6028..cbf4fe99fe5 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/rank-profiles.cfg @@ -1,4 +1,6 @@ rankprofile[].name "default" +rankprofile[].fef.property[].name "query(bar)" +rankprofile[].fef.property[].value "5.5" rankprofile[].fef.property[].name "vespa.type.query.W_0" rankprofile[].fef.property[].value "tensor(hidden[9],x[9])" rankprofile[].fef.property[].name "vespa.type.query.b_0" @@ -12,7 +14,9 @@ rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.b_out" rankprofile[].fef.property[].value "tensor(out[1])" rankprofile[].fef.property[].name "query(b_out)" -rankprofile[].fef.property[].value "{{out:0}:1.0}" +rankprofile[].fef.property[].value "tensor(out[1]):{{out:0}:1.0}" +rankprofile[].fef.property[].name "query(foo)" +rankprofile[].fef.property[].value "5.5" rankprofile[].name "unranked" rankprofile[].fef.property[].name "vespa.rank.firstphase" rankprofile[].fef.property[].value "value(0)" @@ -23,6 +27,8 @@ rankprofile[].fef.property[].value "0" rankprofile[].fef.property[].name "vespa.dump.ignoredefaultfeatures" rankprofile[].fef.property[].value "true" rankprofile[].name "defaultRankProfile" +rankprofile[].fef.property[].name "query(bar)" +rankprofile[].fef.property[].value "5.5" rankprofile[].fef.property[].name "rankingExpression(log10_1p).rankingScript" rankprofile[].fef.property[].value "log10(x + 1)" rankprofile[].fef.property[].name "rankingExpression(textScoreToUse).rankingScript" @@ -84,8 +90,12 @@ rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.b_out" rankprofile[].fef.property[].value "tensor(out[1])" rankprofile[].fef.property[].name "query(b_out)" -rankprofile[].fef.property[].value "{{out:0}:1.0}" +rankprofile[].fef.property[].value "tensor(out[1]):{{out:0}:1.0}" +rankprofile[].fef.property[].name "query(foo)" +rankprofile[].fef.property[].value "5.5" rankprofile[].name "neuralNetworkProfile" +rankprofile[].fef.property[].name "query(bar)" +rankprofile[].fef.property[].value "5.5" rankprofile[].fef.property[].name "rankingExpression(freshnessRank).rankingScript" rankprofile[].fef.property[].value "nativeRank + freshness(createdAt)" rankprofile[].fef.property[].name "rankingExpression(aVoteCountToUse).rankingScript" @@ -189,4 +199,6 @@ rankprofile[].fef.property[].value "tensor(out[9])" rankprofile[].fef.property[].name "vespa.type.query.b_out" rankprofile[].fef.property[].value "tensor(out[1])" rankprofile[].fef.property[].name "query(b_out)" -rankprofile[].fef.property[].value "{{out:0}:1.0}" +rankprofile[].fef.property[].value "tensor(out[1]):{{out:0}:1.0}" +rankprofile[].fef.property[].name "query(foo)" +rankprofile[].fef.property[].value "5.5" diff --git a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg index aa8bf1c7291..524a1253480 100644 --- a/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg +++ b/config-model/src/test/derived/neuralnet_noqueryprofile/schema-info.cfg @@ -79,6 +79,8 @@ schema[].rankprofile[].input[].name "query(W_out)" schema[].rankprofile[].input[].type "tensor(out[9])" schema[].rankprofile[].input[].name "query(b_out)" schema[].rankprofile[].input[].type "tensor(out[1])" +schema[].rankprofile[].input[].name "query(foo)" +schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "unranked" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false @@ -97,6 +99,8 @@ schema[].rankprofile[].input[].name "query(W_out)" schema[].rankprofile[].input[].type "tensor(out[9])" schema[].rankprofile[].input[].name "query(b_out)" schema[].rankprofile[].input[].type "tensor(out[1])" +schema[].rankprofile[].input[].name "query(foo)" +schema[].rankprofile[].input[].type "tensor()" schema[].rankprofile[].name "neuralNetworkProfile" schema[].rankprofile[].hasSummaryFeatures false schema[].rankprofile[].hasRankFeatures false @@ -112,3 +116,5 @@ schema[].rankprofile[].input[].name "query(W_out)" schema[].rankprofile[].input[].type "tensor(out[9])" schema[].rankprofile[].input[].name "query(b_out)" schema[].rankprofile[].input[].type "tensor(out[1])" +schema[].rankprofile[].input[].name "query(foo)" +schema[].rankprofile[].input[].type "tensor()" diff --git a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile b/config-model/src/test/derived/rankprofilemodularity/test/subdirectory/outside_schema2.profile index 8cf3fcfbb78..8cf3fcfbb78 100644 --- a/config-model/src/test/derived/rankprofilemodularity/test/outside_schema2.profile +++ b/config-model/src/test/derived/rankprofilemodularity/test/subdirectory/outside_schema2.profile diff --git a/config-model/src/test/derived/schemainheritance/parent.sd b/config-model/src/test/derived/schemainheritance/parent.sd index 51b11dad444..41c2d89fff5 100644 --- a/config-model/src/test/derived/schemainheritance/parent.sd +++ b/config-model/src/test/derived/schemainheritance/parent.sd @@ -23,10 +23,9 @@ schema parent { indexing: input pf1 | lowercase | index | attribute | summary } rank-profile parent_profile { - } - constant parent_constant { - file: constants/my_constant_tensor_file.json - type: tensor<float>(x{},y{}) + constants { + parent_constant tensor<float>(x{},y{}): file:constants/my_constant_tensor_file.json + } } onnx-model parent_model { file: small_constants_and_functions.onnx diff --git a/config-model/src/test/derived/schemainheritance/ranking-constants.cfg b/config-model/src/test/derived/schemainheritance/ranking-constants.cfg new file mode 100644 index 00000000000..9b34e3f1837 --- /dev/null +++ b/config-model/src/test/derived/schemainheritance/ranking-constants.cfg @@ -0,0 +1,6 @@ +constant[].name "parent_constant" +constant[].fileref "constants/my_constant_tensor_file.json" +constant[].type "tensor<float>(x{},y{})" +constant[].name "child_constant" +constant[].fileref "constants/my_constant_tensor_file.json" +constant[].type "tensor<float>(x{},y{})" diff --git a/config-model/src/test/integration/onnx-model/schemas/test.sd b/config-model/src/test/integration/onnx-model/schemas/test.sd index a15714767ba..82872758dd9 100644 --- a/config-model/src/test/integration/onnx-model/schemas/test.sd +++ b/config-model/src/test/integration/onnx-model/schemas/test.sd @@ -21,14 +21,6 @@ search test { output "path/to/output:0": out } - onnx-model another_model { - file: files/model.onnx - input first_input: attribute(document_field) - input "second/input:0": constant(my_constant) - input "third_input": another_function - output "path/to/output:2": out - } - onnx-model dynamic_model { file: files/dynamic_model.onnx input input: my_function @@ -72,6 +64,13 @@ search test { first-phase { expression: 1 } + onnx-model another_model { + file: files/model.onnx + input first_input: attribute(document_field) + input "second/input:0": constant(my_constant) + input "third_input": another_function + output "path/to/output:2": out + } summary-features { onnx(another_model).out onnx("files/summary_model.onnx", "path/to/output:2") diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java index 3acc44113d0..57a61e40d89 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationDeployTest.java @@ -13,8 +13,8 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.document.DataType; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.DocumentOnlySchema; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.DocumentOnlySchema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.config.ConfigDefinition; import com.yahoo.vespa.config.ConfigDefinitionKey; import com.yahoo.vespa.model.VespaModel; diff --git a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java index 96186150108..273102a00c5 100644 --- a/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java +++ b/config-model/src/test/java/com/yahoo/config/model/ApplicationPackageTester.java @@ -5,7 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.model.application.provider.ApplicationPackageXmlFilesValidator; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import java.io.File; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 323bd73186e..f164a1045b6 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -1554,14 +1554,14 @@ public class ModelProvisioningTest { "<?xml version='1.0' encoding='utf-8' ?>\n" + "<container version='1.0'>" + " <search/>" + - " <nodes jvmargs='xyz' count='3'/>" + + " <nodes jvmargs='-DfooOption=xyz' count='3'/>" + "</container>"; int numberOfHosts = 3; VespaModelTester tester = new VespaModelTester(); tester.addHosts(numberOfHosts); VespaModel model = tester.createModel(services, true); assertEquals(numberOfHosts, model.getRoot().hostSystem().getHosts().size()); - assertEquals("xyz", model.getContainerClusters().get("container").getContainers().get(0).getAssignedJvmOptions()); + assertEquals("-DfooOption=xyz", model.getContainerClusters().get("container").getContainers().get(0).getAssignedJvmOptions()); } @Test diff --git a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java index d4ebbe62507..4a3d85c6a76 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDDocumentTypeTestCase.java @@ -4,11 +4,11 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java index 36a4c7bdafc..4b0d4ee7a9a 100644 --- a/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/document/test/SDFieldTestCase.java @@ -2,8 +2,8 @@ package com.yahoo.document.test; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; import org.junit.Test; import static org.junit.Assert.fail; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/AbstractSchemaTestCase.java index 8ece5cd0fe4..e816456249f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AbstractSchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AbstractSchemaTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.io.IOUtils; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AnnotationReferenceTestCase.java b/config-model/src/test/java/com/yahoo/schema/AnnotationReferenceTestCase.java index 25de24a4315..dbe827ed67f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AnnotationReferenceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AnnotationReferenceTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.DataType; import com.yahoo.document.StructDataType; import com.yahoo.document.Field; import com.yahoo.document.annotation.AnnotationReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; +import com.yahoo.schema.document.SDDocumentType; import com.yahoo.config.model.deploy.TestProperties; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/schema/ArraysTestCase.java index 2ed1ab6892a..aab79617556 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/ArraysTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java b/config-model/src/test/java/com/yahoo/schema/ArraysWeightedSetsTestCase.java index 9c70fcf79f7..57331bea6bd 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ArraysWeightedSetsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/ArraysWeightedSetsTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.ArrayDataType; import com.yahoo.document.CollectionDataType; import com.yahoo.document.DataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -18,7 +18,7 @@ import static org.junit.Assert.assertTrue; */ public class ArraysWeightedSetsTestCase extends AbstractSchemaTestCase { @Test - public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.searchdefinition.parser.ParseException { + public void testArrayWeightedSetsImporting() throws java.io.IOException, com.yahoo.schema.parser.ParseException { Schema schema = ApplicationBuilder.buildFromFile("src/test/examples/arraysweightedsets.sd"); SDField tags = (SDField) schema.getDocument().getField("tags"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java index 92d72677086..bbe63f95787 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeSettingsTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.StructDataType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.IndexingScript; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.IndexingScript; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.tensor.TensorType; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java index 5d91d7df3ca..fb7d68faf2d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/AttributeUtils.java +++ b/config-model/src/test/java/com/yahoo/schema/AttributeUtils.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDField; /** * Convenience class for tests that need to set attribute properties on fields. diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java b/config-model/src/test/java/com/yahoo/schema/CommentTestCase.java index 3e92c1df9e7..4df4ead171b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/CommentTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/CommentTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java index ca7313a5c3f..482bc877081 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DiversityTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/DiversityTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.search.query.ranking.Diversity; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static org.junit.Assert.fail; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java index 5bbb751585b..ef4d8e05540 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentGraphValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/DocumentGraphValidatorTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java index fe1b19be64b..1592060f466 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/DocumentReferenceResolverTest.java +++ b/config-model/src/test/java/com/yahoo/schema/DocumentReferenceResolverTest.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/FeatureNamesTestCase.java index 3494ab780b2..bff4b434408 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FeatureNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/FeatureNamesTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import org.junit.Ignore; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java b/config-model/src/test/java/com/yahoo/schema/FieldOfTypeDocumentTestCase.java index fdd7fe95c45..4a590288d53 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/FieldOfTypeDocumentTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/FieldOfTypeDocumentTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; @@ -7,7 +7,7 @@ import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.DocumentTypeManagerConfigurer; import com.yahoo.document.Field; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import org.junit.Test; import java.io.IOException; import java.util.ArrayList; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java index f486126ba4f..92d11b3a18a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ImportedFieldsEnumeratorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/ImportedFieldsEnumeratorTest.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; -import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; import org.junit.Test; import java.util.HashSet; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java b/config-model/src/test/java/com/yahoo/schema/IncorrectRankingExpressionFileRefTestCase.java index 23be59e38f5..87e168adb66 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectRankingExpressionFileRefTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/IncorrectRankingExpressionFileRefTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.parser.ParseException; import com.yahoo.yolean.Exceptions; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/IncorrectSummaryTypesTestCase.java index d81d07bf6e2..e58cce6472a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IncorrectSummaryTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/IncorrectSummaryTypesTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java b/config-model/src/test/java/com/yahoo/schema/IndexSettingsTestCase.java index 198f8b6c79e..6082372b428 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexSettingsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/IndexSettingsTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java b/config-model/src/test/java/com/yahoo/schema/IndexingParsingTestCase.java index 8ba7dfdb14f..6a51000fffe 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/IndexingParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/IndexingParsingTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static org.junit.Assert.assertNotNull; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/MultipleSummariesTestCase.java index 317b860d7d2..6d6249dc372 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/MultipleSummariesTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java b/config-model/src/test/java/com/yahoo/schema/NameFieldCheckTestCase.java index 1e766b7793d..9b4b6864309 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/NameFieldCheckTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/NameFieldCheckTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java b/config-model/src/test/java/com/yahoo/schema/OutsideTestCase.java index 6563fb8d3f1..0c0684e23e3 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/OutsideTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/OutsideTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java b/config-model/src/test/java/com/yahoo/schema/PredicateDataTypeTestCase.java index 2b49f3d67a6..dbea8fb8aeb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/PredicateDataTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/PredicateDataTypeTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.ImmutableSDField; +import com.yahoo.schema.document.ImmutableSDField; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java b/config-model/src/test/java/com/yahoo/schema/RankProfileRegistryTest.java index d07f4513c3c..de061defb87 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileRegistryTest.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileRegistryTest.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.model.test.MockApplicationPackage; @@ -36,7 +36,7 @@ public class RankProfileRegistryTest { public void testRankProfileDuplicateNameIsIllegal() { Schema schema = new Schema("foo", MockApplicationPackage.createEmpty()); RankProfileRegistry rankProfileRegistry = RankProfileRegistry.createRankProfileRegistryWithBuiltinRankProfiles(schema); - RankProfile barRankProfile = new RankProfile("bar", schema, rankProfileRegistry, schema.rankingConstants()); + RankProfile barRankProfile = new RankProfile("bar", schema, rankProfileRegistry); rankProfileRegistry.add(barRankProfile); rankProfileRegistry.add(barRankProfile); } @@ -48,7 +48,7 @@ public class RankProfileRegistryTest { for (String rankProfileName : RankProfileRegistry.overridableRankProfileNames) { assertNull(rankProfileRegistry.get(schema, rankProfileName).getFunctions().get("foo")); - RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, schema, rankProfileRegistry, schema.rankingConstants()); + RankProfile rankProfileWithAddedFunction = new RankProfile(rankProfileName, schema, rankProfileRegistry); rankProfileWithAddedFunction.addFunction(new ExpressionFunction("foo", RankingExpression.from("1+2")), true); rankProfileRegistry.add(rankProfileWithAddedFunction); assertNotNull(rankProfileRegistry.get(schema, rankProfileName).getFunctions().get("foo")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java index 03b504bb821..c66d44556ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankProfileTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; @@ -14,12 +14,12 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -52,7 +52,7 @@ public class RankProfileTestCase extends AbstractSchemaTestCase { a.setRankType(RankType.IDENTITY); document.addField("b", DataType.STRING); schema.addDocument(document); - RankProfile child = new RankProfile("child", schema, rankProfileRegistry, schema.rankingConstants()); + RankProfile child = new RankProfile("child", schema, rankProfileRegistry); child.inherit("default"); rankProfileRegistry.add(child); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java index 6de890549c1..c3595717220 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankPropertiesTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java b/config-model/src/test/java/com/yahoo/schema/RankingConstantTest.java index 6eb74dede62..883e6b50abb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingConstantTest.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingConstantTest.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; @@ -30,7 +30,7 @@ public class RankingConstantTest { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " rank-profile my_rank_profile {", " first-phase {", @@ -46,12 +46,12 @@ public class RankingConstantTest { schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); - Iterator<RankingConstant> constantIterator = schema.rankingConstants().asMap().values().iterator(); - RankingConstant constant = constantIterator.next(); - assertEquals(TENSOR_NAME, constant.getName()); - assertEquals(TENSOR_FILE, constant.getFileName()); - assertEquals(TENSOR_TYPE, constant.getType()); - assertEquals(RankingConstant.PathType.FILE, constant.getPathType()); + Iterator<RankProfile.Constant> constantIterator = schema.constants().values().iterator(); + RankProfile.Constant constant = constantIterator.next(); + assertEquals(TENSOR_NAME, constant.name().simpleArgument().get()); + assertEquals(TENSOR_FILE, constant.valuePath().get()); + assertEquals(TENSOR_TYPE, constant.type().toString()); + assertEquals(DistributableResource.PathType.FILE, constant.pathType().get()); assertFalse(constantIterator.hasNext()); } @@ -63,7 +63,7 @@ public class RankingConstantTest { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must have a type"); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " file: bar.baz", @@ -79,7 +79,7 @@ public class RankingConstantTest { thrown.expect(IllegalArgumentException.class); thrown.expectMessage("must have a file"); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x[])", @@ -93,7 +93,7 @@ public class RankingConstantTest { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x{})", @@ -103,8 +103,8 @@ public class RankingConstantTest { )); schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); - RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); - assertEquals("simplename", constant.getFileName()); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals("simplename", constant.valuePath().get()); } @Test @@ -112,7 +112,7 @@ public class RankingConstantTest { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x{})", @@ -122,9 +122,9 @@ public class RankingConstantTest { )); schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); - RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); - assertEquals(RankingConstant.PathType.URI, constant.getPathType()); - assertEquals("http://somewhere.far.away/in/another-galaxy", constant.getUri()); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("http://somewhere.far.away/in/another-galaxy", constant.valuePath().get()); } @Test @@ -132,7 +132,7 @@ public class RankingConstantTest { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x{})", @@ -142,9 +142,9 @@ public class RankingConstantTest { )); schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); - RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); - assertEquals(RankingConstant.PathType.URI, constant.getPathType()); - assertEquals("https://somewhere.far.away:4443/in/another-galaxy", constant.getUri()); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("https://somewhere.far.away:4443/in/another-galaxy", constant.valuePath().get()); } @Test @@ -152,7 +152,7 @@ public class RankingConstantTest { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x{})", @@ -162,9 +162,9 @@ public class RankingConstantTest { )); schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); - RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); - assertEquals(RankingConstant.PathType.URI, constant.getPathType()); - assertEquals("http://somewhere.far.away:4080/in/another-galaxy", constant.getUri()); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("http://somewhere.far.away:4080/in/another-galaxy", constant.valuePath().get()); } @Test @@ -172,7 +172,7 @@ public class RankingConstantTest { RankProfileRegistry rankProfileRegistry = new RankProfileRegistry(); ApplicationBuilder schemaBuilder = new ApplicationBuilder(rankProfileRegistry); schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x{})", @@ -182,9 +182,9 @@ public class RankingConstantTest { )); schemaBuilder.build(true); Schema schema = schemaBuilder.getSchema(); - RankingConstant constant = schema.rankingConstants().asMap().values().iterator().next(); - assertEquals(RankingConstant.PathType.URI, constant.getPathType()); - assertEquals("http:somewhere.far.away/in/another-galaxy", constant.getUri()); + RankProfile.Constant constant = schema.constants().values().iterator().next(); + assertEquals(DistributableResource.PathType.URI, constant.pathType().get()); + assertEquals("http:somewhere.far.away/in/another-galaxy", constant.valuePath().get()); } @Test @@ -196,7 +196,7 @@ public class RankingConstantTest { "<URI_PATH> ..."; try { schemaBuilder.addSchema(joinLines( - "search test {", + "schema test {", " document test { }", " constant foo {", " type: tensor(x{})", diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionConstantsTestCase.java index cd98ca96c42..bd0bd65295c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionConstantsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionConstantsTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.collections.Pair; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -7,9 +7,9 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import com.yahoo.yolean.Exceptions; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.util.List; @@ -37,7 +37,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " rank-profile parent {\n" + " constants {\n" + " p1 double: 7 \n" + - " p2 double: 0 \n" + + " constant(p2) double: 0 \n" + " }\n" + " first-phase {\n" + " expression: p2 * (1.3 + p1 )\n" + @@ -52,7 +52,7 @@ public class RankingExpressionConstantsTestCase extends AbstractSchemaTestCase { " }\n" + " constants {\n" + " a: 1.0 \n" + - " b: 2 \n" + + " constant(b): 2 \n" + " c: 3.5 \n" + " }\n" + " }\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java index fd35c19906d..0695e20d780 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionInliningTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionInliningTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.collections.Pair; import com.yahoo.config.application.api.DeployLogger; @@ -7,9 +7,9 @@ import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionLoopDetectionTestCase.java index ff8a9df4076..dd69fb6c591 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionLoopDetectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionLoopDetectionTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import com.yahoo.yolean.Exceptions; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionShadowingTestCase.java index 8354eb59416..250879b1570 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionShadowingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionShadowingTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.collections.Pair; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -8,9 +8,9 @@ import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/RankingExpressionValidationTestCase.java index ee3b4ab25e0..e42acee9bed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/RankingExpressionValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/RankingExpressionValidationTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.parser.ParseException; import com.yahoo.yolean.Exceptions; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/ReservedWordsAsFieldNamesTestCase.java index a8aaee83938..df9d4a63650 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/ReservedWordsAsFieldNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/ReservedWordsAsFieldNamesTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java index 652a06a6025..45780d39021 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SDDocumentTypeOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SDDocumentTypeOrdererTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; import com.yahoo.document.DataTypeName; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDDocumentType; -import com.yahoo.searchdefinition.document.TemporarySDField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDDocumentType; +import com.yahoo.schema.document.TemporarySDField; import org.junit.Test; import java.util.ArrayList; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java index 9c895b452d5..e93dd0e0a8f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaImporterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaImporterTestCase.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.RankType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.MakeAliases; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.RankType; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.MakeAliases; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaParsingTestCase.java index 6d619b76c18..8fe691db802 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaParsingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaParsingTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import java.io.IOException; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java index 4b40da9e289..67d8ce4ff78 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SchemaTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.ImportedFieldsResolver; -import com.yahoo.searchdefinition.processing.OnnxModelTypeResolver; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.ImportedFieldsResolver; +import com.yahoo.schema.processing.OnnxModelTypeResolver; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; @@ -122,10 +122,9 @@ public class SchemaTestCase { " indexing: input pf1 | lowercase | index | attribute | summary" + " }" + " rank-profile child1_profile inherits parent_profile {" + - " }" + - " constant child1_constant {" + - " file: constants/my_constant_tensor_file.json" + - " type: tensor<float>(x{},y{})" + + " constants {" + + " child1_constant tensor<float>(x{},y{}): file:constants/my_constant_tensor_file.json" + + " }" + " }" + " onnx-model child1_model {" + " file: models/my_model.onnx" + @@ -188,15 +187,17 @@ public class SchemaTestCase { assertNotNull(child1.getExtraField("child1_field")); assertNotNull(builder.getRankProfileRegistry().get(child1, "parent_profile")); assertNotNull(builder.getRankProfileRegistry().get(child1, "child1_profile")); + var child1profile = builder.getRankProfileRegistry().get(child1, "child1_profile"); assertEquals("parent_profile", builder.getRankProfileRegistry().get(child1, "child1_profile").inheritedNames().get(0)); - assertNotNull(child1.rankingConstants().get("parent_constant")); - assertNotNull(child1.rankingConstants().get("child1_constant")); - assertTrue(child1.rankingConstants().asMap().containsKey("parent_constant")); - assertTrue(child1.rankingConstants().asMap().containsKey("child1_constant")); + assertNotNull(child1.constants().get(FeatureNames.asConstantFeature("parent_constant"))); + assertNotNull(child1profile.constants().get(FeatureNames.asConstantFeature("child1_constant"))); + assertTrue(child1.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); + assertTrue(child1profile.constants().containsKey(FeatureNames.asConstantFeature("child1_constant"))); + assertTrue(child1profile.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); assertNotNull(child1.onnxModels().get("parent_model")); assertNotNull(child1.onnxModels().get("child1_model")); - assertTrue(child1.onnxModels().asMap().containsKey("parent_model")); - assertTrue(child1.onnxModels().asMap().containsKey("child1_model")); + assertTrue(child1.onnxModels().containsKey("parent_model")); + assertTrue(child1.onnxModels().containsKey("child1_model")); assertNotNull(child1.getSummary("parent_summary")); assertNotNull(child1.getSummary("child1_summary")); assertEquals("parent_summary", child1.getSummary("child1_summary").inherited().get().getName()); @@ -224,14 +225,14 @@ public class SchemaTestCase { assertNotNull(builder.getRankProfileRegistry().get(child2, "parent_profile")); assertNotNull(builder.getRankProfileRegistry().get(child2, "child2_profile")); assertEquals("parent_profile", builder.getRankProfileRegistry().get(child2, "child2_profile").inheritedNames().get(0)); - assertNotNull(child2.rankingConstants().get("parent_constant")); - assertNotNull(child2.rankingConstants().get("child2_constant")); - assertTrue(child2.rankingConstants().asMap().containsKey("parent_constant")); - assertTrue(child2.rankingConstants().asMap().containsKey("child2_constant")); + assertNotNull(child2.constants().get(FeatureNames.asConstantFeature("parent_constant"))); + assertNotNull(child2.constants().get(FeatureNames.asConstantFeature("child2_constant"))); + assertTrue(child2.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); + assertTrue(child2.constants().containsKey(FeatureNames.asConstantFeature("child2_constant"))); assertNotNull(child2.onnxModels().get("parent_model")); assertNotNull(child2.onnxModels().get("child2_model")); - assertTrue(child2.onnxModels().asMap().containsKey("parent_model")); - assertTrue(child2.onnxModels().asMap().containsKey("child2_model")); + assertTrue(child2.onnxModels().containsKey("parent_model")); + assertTrue(child2.onnxModels().containsKey("child2_model")); assertNotNull(child2.getSummary("parent_summary")); assertNotNull(child2.getSummary("child2_summary")); assertEquals("parent_summary", child2.getSummary("child2_summary").inherited().get().getName()); @@ -317,8 +318,8 @@ public class SchemaTestCase { builder.build(true); var application = builder.application(); - assertInheritedFromParent(application.schemas().get("child"), application, builder.getRankProfileRegistry()); - assertInheritedFromParent(application.schemas().get("grandchild"), application, builder.getRankProfileRegistry()); + assertInheritedFromParent(application.schemas().get("child"), builder.getRankProfileRegistry()); + assertInheritedFromParent(application.schemas().get("grandchild"), builder.getRankProfileRegistry()); } @Test @@ -419,17 +420,17 @@ public class SchemaTestCase { } } - private void assertInheritedFromParent(Schema schema, Application application, RankProfileRegistry rankProfileRegistry) { + private void assertInheritedFromParent(Schema schema, RankProfileRegistry rankProfileRegistry) { assertEquals("pf1", schema.fieldSets().userFieldSets().get("parent_set").getFieldNames().stream().findFirst().get()); assertEquals(Stemming.NONE, schema.getStemming()); assertEquals(Stemming.BEST, schema.getIndex("parent_index").getStemming()); assertNotNull(schema.getField("parent_field")); assertNotNull(schema.getExtraField("parent_field")); assertNotNull(rankProfileRegistry.get(schema, "parent_profile")); - assertNotNull(schema.rankingConstants().get("parent_constant")); - assertTrue(schema.rankingConstants().asMap().containsKey("parent_constant")); + assertNotNull(schema.constants().get(FeatureNames.asConstantFeature("parent_constant"))); + assertTrue(schema.constants().containsKey(FeatureNames.asConstantFeature("parent_constant"))); assertNotNull(schema.onnxModels().get("parent_model")); - assertTrue(schema.onnxModels().asMap().containsKey("parent_model")); + assertTrue(schema.onnxModels().containsKey("parent_model")); assertNotNull(schema.getSummary("parent_summary")); assertTrue(schema.getSummaries().containsKey("parent_summary")); assertNotNull(schema.getSummaryField("pf1")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java b/config-model/src/test/java/com/yahoo/schema/StemmingSettingTestCase.java index aec258f2df0..5dd75166783 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StemmingSettingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/StemmingSettingTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java b/config-model/src/test/java/com/yahoo/schema/StructTestCase.java index 078eccc166f..b140892ed5e 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/StructTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/StructTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.derived.Deriver; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.Deriver; +import com.yahoo.schema.parser.ParseException; import org.junit.Ignore; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java index 086e3485962..0fab9b381fc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/SummaryTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import com.yahoo.vespa.objects.FieldBase; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/UrlFieldValidationTestCase.java index f2d78796553..83bb0d4548c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/UrlFieldValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/UrlFieldValidationTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition; +package com.yahoo.schema; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import com.yahoo.yolean.Exceptions; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java index 8548e50d4bc..ad2c2d6078d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AbstractExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AbstractExportingTestCase.java @@ -1,17 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; +import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; -import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; @@ -49,12 +48,11 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { ApplicationBuilder builder, DeployLogger logger) throws IOException { DerivedConfiguration config = new DerivedConfiguration(builder.getSchema(schemaName), - logger, - properties, - builder.getRankProfileRegistry(), - builder.getQueryProfileRegistry(), - new ImportedMlModels(), - new InThreadExecutorService()); + new DeployState.Builder().properties(properties) + .deployLogger(logger) + .rankProfileRegistry(builder.getRankProfileRegistry()) + .queryProfiles(builder.getQueryProfileRegistry()) + .build()); return export(dirName, builder, config); } @@ -71,6 +69,7 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { .produce(builder.getModel(), new DocumentmanagerConfig.Builder()), path); DerivedConfiguration.exportDocuments(new DocumentTypes().produce(builder.getModel(), new DocumenttypesConfig.Builder()), path); DerivedConfiguration.exportQueryProfiles(builder.getQueryProfileRegistry(), path); + config.exportConstants(path); return config; } @@ -108,10 +107,10 @@ public abstract class AbstractExportingTestCase extends AbstractSchemaTestCase { } protected DerivedConfiguration assertCorrectDeriving(String dirName, - String searchDefinitionName, + String schemaName, TestProperties properties, DeployLogger logger) throws IOException, ParseException { - DerivedConfiguration derived = derive(dirName, searchDefinitionName, properties, logger); + DerivedConfiguration derived = derive(dirName, schemaName, properties, logger); assertCorrectConfigFiles(dirName); return derived; } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AnnotationsTestCase.java index bfbf539074c..60867261f93 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AnnotationsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AnnotationsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ArraysTestCase.java index c2bd86ba5f7..5b138413a7a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ArraysTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ArraysTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java index a272ab14dad..1c51d3ec365 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributeListTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AttributeListTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/AttributesTestCase.java index 2b345bdb158..3d08805acdf 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/AttributesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/AttributesTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/CasingTestCase.java index ee9a9eac02c..80ebcb825f4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CasingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/CasingTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/CombinedAttributeAndIndexSchemaTestCase.java index bb339688fcb..542320d9670 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/CombinedAttributeAndIndexSchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/CombinedAttributeAndIndexSchemaTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/DeriverTestCase.java index b1fb1962fc8..422f4522b26 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DeriverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/DeriverTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.DataType; import com.yahoo.document.DocumentTypeManager; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; +import com.yahoo.schema.AbstractSchemaTestCase; import org.junit.Test; import java.util.List; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DuplicateStructTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/DuplicateStructTestCase.java index ff68e47a1e9..7915a1d7763 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/DuplicateStructTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/DuplicateStructTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/EmptyRankProfileTestCase.java index a3123550efa..c3195d1a626 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/EmptyRankProfileTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/EmptyRankProfileTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import org.junit.Test; /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExactMatchTestCase.java index 45e65bb90f9..13a0a8201fb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExactMatchTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java index ac97863b7a7..16a9a459dcb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExportingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExportingTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExpressionsAsArgsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ExpressionsAsArgsTestCase.java index 733786e2c71..d2020305bc1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ExpressionsAsArgsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ExpressionsAsArgsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/FieldsetTestCase.java index 3877b351ac8..fdab49c9fff 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/FieldsetTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/FieldsetTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/GeminiTestCase.java index 07e6fbf7b1b..5531fb65942 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/GeminiTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/GeminiTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.collections.Pair; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -19,7 +19,7 @@ public class GeminiTestCase extends AbstractExportingTestCase { @Test public void testRanking2() throws IOException, ParseException { DerivedConfiguration c = assertCorrectDeriving("gemini2"); - RawRankProfile p = c.getRankProfileList().getRankProfile("test"); + RawRankProfile p = c.getRankProfileList().getRankProfiles().get("test"); Map<String, String> ranking = removePartKeySuffixes(asMap(p.configProperties())); assertEquals("attribute(right)", resolve(lookup("toplevel", ranking), ranking)); } diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java index ba485b7b96b..1e57d52e3b0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IdTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/IdTestCase.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ImportedFieldsTestCase.java index 5eeae347c88..5578a1a602b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ImportedFieldsTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java index 17f85840af8..1f40c6bcb50 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IndexSchemaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/IndexSchemaTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.document.DataType; import com.yahoo.document.Field; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java index 9f691be2956..1c433237fd8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/InheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/InheritanceTestCase.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Index; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import org.junit.Rule; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/IntegerAttributeToStringIndexTestCase.java index 0980f766579..b5f222673ab 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/IntegerAttributeToStringIndexTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/IntegerAttributeToStringIndexTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java index 1e7dd3a2405..c5090c88a1b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LiteralBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/LiteralBoostTestCase.java @@ -1,24 +1,24 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; import java.util.Arrays; import java.util.Set; -import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; import static org.junit.Assert.assertTrue; /** @@ -38,7 +38,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { SDField field1 = document.addField("a", DataType.STRING); field1.parseIndexingScript("{ index }"); field1.setLiteralBoost(20); - RankProfile other = new RankProfile("other", schema, rankProfileRegistry, schema.rankingConstants()); + RankProfile other = new RankProfile("other", schema, rankProfileRegistry); rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); @@ -70,7 +70,7 @@ public class LiteralBoostTestCase extends AbstractExportingTestCase { schema.addDocument(document); SDField field1 = document.addField("a", DataType.STRING); field1.parseIndexingScript("{ index }"); - RankProfile other = new RankProfile("other", schema, rankProfileRegistry, schema.rankingConstants()); + RankProfile other = new RankProfile("other", schema, rankProfileRegistry); rankProfileRegistry.add(other); other.addRankSetting(new RankProfile.RankSetting("a", RankProfile.RankSetting.Type.LITERALBOOST, 333)); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/LowercaseTestCase.java index 5427c6a19e7..f234a9cc324 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/LowercaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/LowercaseTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MailTestCase.java index 53bf55fc73f..c48c44554ed 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MailTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/MailTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MatchSettingsResolvingTestCase.java index 259a7ab8804..a7df862134a 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MatchSettingsResolvingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/MatchSettingsResolvingTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MultiStructTestCase.java index 4afbb4c5436..66b3698b38c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultiStructTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/MultiStructTestCase.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.schema.ApplicationBuilder; import org.junit.Test; /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/MultipleSummariesTestCase.java index 245049fd4bc..d434673e43a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/MultipleSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/MultipleSummariesTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NameCollisionTestCase.java index c8df63e1028..689ff9814cc 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NameCollisionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NameCollisionTestCase.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DocumentTypeManager; -import com.yahoo.searchdefinition.ApplicationBuilder; import org.junit.Test; import static org.junit.Assert.assertThrows; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NativeRankTypeDefinitionsTestCase.java index 8c6bc40ece0..f628420556a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NativeRankTypeDefinitionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NativeRankTypeDefinitionsTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.RankType; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.RankType; import org.junit.Test; import java.util.Iterator; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java index 16ec896ca66..baee7bec2a2 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NearestNeighborTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NearestNeighborTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.component.ComponentId; import com.yahoo.search.Query; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java index c7b0d131eb6..6e584099331 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NeuralNetTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NeuralNetTestCase.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.search.Query; -import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NuwaTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/NuwaTestCase.java index 75b8d3bf584..210c8a9bdd4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/NuwaTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/NuwaTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Ignore; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/OrderIlscriptsTestCase.java index 1bb8083e7b2..8af0d0a21d3 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/OrderIlscriptsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/OrderIlscriptsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/PrefixExactAttributeTestCase.java index 0b571d11917..fdcb71432e4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/PrefixExactAttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/PrefixExactAttributeTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java index 3a900e0840a..a83db0caf5a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/RankProfilesTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/RankPropertiesTestCase.java index 0f29d2dda40..8db880e56fe 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/RankPropertiesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/RankPropertiesTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFieldsTestCase.java index 086a69a8b20..99d0cf8bf6d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFieldsTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFromSeveralTestCase.java index dbbe1497ffe..ff4506a7f57 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/ReferenceFromSeveralTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/ReferenceFromSeveralTestCase.java @@ -1,10 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.schema.ApplicationBuilder; import org.junit.Test; /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaInheritanceTestCase.java index a458036a03f..1b5d55158b5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaInheritanceTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.io.IOUtils; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.File; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java index 11e21c7915d..e672763f13c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SchemaOrdererTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SchemaOrdererTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; import org.junit.Test; import java.util.Arrays; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SimpleInheritTestCase.java index d89e5f2c957..d8b39bfd978 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SimpleInheritTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SimpleInheritTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.File; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SliceTestCase.java index e6c7efd7052..2aad47dae6c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SliceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SliceTestCase.java @@ -1,12 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.component.ComponentId; -import com.yahoo.search.Query; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; -import com.yahoo.search.query.profile.config.QueryProfileConfigurer; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.vespa.model.container.search.QueryProfiles; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SortingTestCase.java index 2516b1b0d9a..0c091a7a367 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SortingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SortingTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/StreamingStructTestCase.java index f9bd3033e14..6f27930e239 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StreamingStructTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/StreamingStructTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/StructAnyOrderTestCase.java index 3338f6cf87d..865b5da87cf 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructAnyOrderTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/StructAnyOrderTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/StructInheritanceTestCase.java index 1f6c70c9383..092e64420e8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/StructInheritanceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/StructInheritanceTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.schema.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java index 6abe0e763cb..f0fc58b97e5 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryMapTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryMapTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.test.MockApplicationPackage; -import com.yahoo.searchdefinition.*; +import com.yahoo.schema.*; import com.yahoo.vespa.config.search.SummarymapConfig; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; +import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java index fd5caeb6b6d..341d3ef7d43 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/SummaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/SummaryTestCase.java @@ -1,11 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.config.search.SummaryConfig; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java b/config-model/src/test/java/com/yahoo/schema/derived/TestableDeployLogger.java index 52d57bd0468..bcde0f6b544 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TestableDeployLogger.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TestableDeployLogger.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.application.api.DeployLogger; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TokenizationTestCase.java index 4b9d5386314..ac6acf172e9 100755 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TokenizationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TokenizationTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TwoStreamingStructsTestCase.java index 4e629fcb4d8..72411fa1770 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TwoStreamingStructsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TwoStreamingStructsTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java index 62a79e49146..84a561924ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypeConversionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TypeConversionTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.processing.Processing; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.processing.Processing; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/TypesTestCase.java index cee6d4f95c8..7443ef01c95 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/TypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/TypesTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java index 9855ca30ebc..c59f82a2c12 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/derived/VsmFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/derived/VsmFieldsTestCase.java @@ -1,15 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.derived; +package com.yahoo.schema.derived; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; import com.yahoo.vespa.config.search.vsm.VsmfieldsConfig; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java b/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java index 99692e70041..ea3f207df91 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/ComplexAttributeFieldUtilsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/document/ComplexAttributeFieldUtilsTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static com.yahoo.config.model.test.TestUtil.joinLines; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java b/config-model/src/test/java/com/yahoo/schema/document/HnswIndexParamsTestCase.java index d6ff8fa1067..8ef51369ecb 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/document/HnswIndexParamsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/document/HnswIndexParamsTestCase.java @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.document; +package com.yahoo.schema.document; import java.util.Optional; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/ConvertIntermediateTestCase.java index 127f5184c4c..516c259013f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ConvertIntermediateTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/ConvertIntermediateTestCase.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.document.DataType; -import com.yahoo.document.DocumentType; import com.yahoo.document.DocumentTypeManager; import static com.yahoo.config.model.test.TestUtil.joinLines; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java index e29e4833856..c4ee1d27c8c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateCollectionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/IntermediateCollectionTestCase.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; -import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import static com.yahoo.config.model.test.TestUtil.joinLines; @@ -106,7 +105,7 @@ public class IntermediateCollectionTestCase { var collection = new IntermediateCollection(); collection.addSchemaFromFile("src/test/derived/rankprofilemodularity/test.sd"); collection.addRankProfileFile("test", "src/test/derived/rankprofilemodularity/test/outside_schema1.profile"); - collection.addRankProfileFile("test", readerOf("src/test/derived/rankprofilemodularity/test/outside_schema2.profile")); + collection.addRankProfileFile("test", readerOf("src/test/derived/rankprofilemodularity/test/subdirectory/outside_schema2.profile")); var schemes = collection.getParsedSchemas(); assertEquals(schemes.size(), 1); var schema = schemes.get("test"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/ParsedDocumentTestCase.java index 9ae7b3aa4b8..9245b64b09e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/ParsedDocumentTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/ParsedDocumentTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java index 1e0c554af81..d4e4f1dbb88 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/parser/IntermediateParserTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/parser/SchemaParserTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.parser; +package com.yahoo.schema.parser; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.deploy.TestProperties; @@ -10,20 +10,22 @@ import java.io.File; import org.junit.Test; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertThrows; /** * @author arnej */ -public class IntermediateParserTestCase { +public class SchemaParserTestCase { ParsedSchema parseString(String input) throws Exception { var deployLogger = new BaseDeployLogger(); var modelProperties = new TestProperties(); var stream = new SimpleCharStream(input); try { - var parser = new IntermediateParser(stream, deployLogger, modelProperties); + var parser = new SchemaParser(stream, deployLogger, modelProperties); return parser.schema(); } catch (ParseException pe) { throw new ParseException(stream.formatException(pe.getMessage())); @@ -60,7 +62,7 @@ public class IntermediateParserTestCase { } @Test - public void multiple_documents_disallowed() throws Exception { + public void multiple_documents_disallowed() { String input = joinLines ("schema foo {", " document foo {", @@ -85,13 +87,13 @@ public class IntermediateParserTestCase { } void checkFileParses(String fileName) throws Exception { - System.err.println("TRY parsing: "+fileName); var schema = parseFile(fileName); - assertTrue(schema != null); - assertTrue(schema.name() != null); - assertTrue(! schema.name().equals("")); + assertNotNull(schema); + assertNotNull(schema.name()); + assertNotEquals("", schema.name()); } + // TODO: Many (all)? of the files below are parsed from other tests and can be removed from here @Test public void parse_various_old_sdfiles() throws Exception { checkFileParses("src/test/cfg/search/data/travel/schemas/TTData.sd"); @@ -179,7 +181,6 @@ public class IntermediateParserTestCase { checkFileParses("src/test/derived/namecollision/collision.sd"); checkFileParses("src/test/derived/namecollision/collisionstruct.sd"); checkFileParses("src/test/derived/nearestneighbor/test.sd"); - checkFileParses("src/test/derived/neuralnet/neuralnet.sd"); checkFileParses("src/test/derived/newrank/newrank.sd"); checkFileParses("src/test/derived/nuwa/newsindex.sd"); checkFileParses("src/test/derived/orderilscripts/orderilscripts.sd"); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java b/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java index 0adaba4cf68..0d64dd5c953 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AddAttributeTransformToSummaryOfImportedFieldsTest.java @@ -1,20 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.ImportedSimpleField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java index 926e26451d7..103d08b39a8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AdjustPositionSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AdjustPositionSummaryFieldsTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.document.DataType; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java index 8a0c1e2fe71..82650598f29 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertIndexingScript.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertIndexingScript.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.IndexingScript; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.IndexingScript; import com.yahoo.vespa.indexinglanguage.expressions.Expression; import com.yahoo.vespa.indexinglanguage.parser.ParseException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java index 534c82157e4..0b4d7c3a2b6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AssertSearchBuilder.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AssertSearchBuilder.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java index e30ac8f8e00..40ebe458c74 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/AttributesExactMatchTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/AttributesExactMatchTestCase.java @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java index efec9206ed9..c37bc8085c7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoldingTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoldingTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static org.junit.Assert.assertEquals; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java index db27bbbb84d..287cc6559d1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/BoolAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/BoolAttributeValidatorTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; +import static com.yahoo.schema.ApplicationBuilder.createFromString; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java index c0124ca0386..1956b87a689 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DictionaryTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/DictionaryTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.test.TestUtil; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.Case; -import com.yahoo.searchdefinition.document.Dictionary; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.Case; +import com.yahoo.schema.document.Dictionary; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.config.search.AttributesConfig; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java index b03aff455c5..64b0a437b1d 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/DisallowComplexMapAndWsetKeyTypesTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java b/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java index 0c25cef49a1..b249b407c7b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/FastAccessValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/FastAccessValidatorTest.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.test.TestUtil; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java index 833a6effe4a..594124c9500 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSchemaFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSchemaFieldsTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.parser.ParseException; +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java index 443eff07f41..111ed266d74 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitStructTypesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitStructTypesTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.document.*; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java index cc2bc7d7bf6..50deb5d5b42 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummariesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummariesTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java index 175b8d6fe1e..f32c9079d36 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImplicitSummaryFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImplicitSummaryFieldsTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.documentmodel.DocumentSummary; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java index d1d3f4489ce..5baa64d06d4 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsResolverTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.document.DataType; import com.yahoo.document.TensorDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.ImmutableImportedSDField; -import com.yahoo.searchdefinition.document.ImmutableSDField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporarySDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.ImmutableImportedSDField; +import com.yahoo.schema.document.ImmutableSDField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporarySDField; import com.yahoo.tensor.TensorType; import org.junit.Rule; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java index b15b81b717d..ab702154527 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ImportedFieldsTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; - -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.document.ImportedComplexField; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.parser.ParseException; +package com.yahoo.schema.processing; + +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.document.ImportedComplexField; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.parser.ParseException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java index 725b1b17ff0..71c79feedc1 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingInputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingInputsTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; /** * @author Simon Thoresen Hult diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java index 5ea4b37991a..687549f920e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingOutputsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingOutputsTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java index 9e4ba1c6728..76cb6a5505c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingScriptRewriterTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingScriptRewriterTestCase.java @@ -1,17 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; -import com.yahoo.searchdefinition.Index; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.BooleanIndexDefinition; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Index; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.BooleanIndexDefinition; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; @@ -24,7 +24,7 @@ import java.util.OptionalInt; import java.util.OptionalLong; import java.util.Set; -import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; import static org.junit.Assert.assertEquals; /** diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java index 74a8cdfdb6a..4da6880aa26 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValidationTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValidationTestCase.java @@ -1,16 +1,16 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.AbstractExportingTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; import java.util.Arrays; -import static com.yahoo.searchdefinition.processing.AssertIndexingScript.assertIndexing; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertIndexingScript.assertIndexing; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; /** * @author Simon Thoresen Hult diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java index 25d966b1324..2784fe69b28 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IndexingValuesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IndexingValuesTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuilds; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuilds; /** * @author Simon Thoresen Hult diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java index 2465839b72b..f36effab146 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/IntegerIndex2AttributeTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/IntegerIndex2AttributeTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java index 05d816ed716..530b6a95ce8 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchPhaseSettingsValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchPhaseSettingsValidatorTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import org.junit.Test; -import static com.yahoo.searchdefinition.processing.AssertSearchBuilder.assertBuildFails; +import static com.yahoo.schema.processing.AssertSearchBuilder.assertBuildFails; public class MatchPhaseSettingsValidatorTestCase { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java index b48927d58a3..c401376ac3a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/MatchedElementsOnlyResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/MatchedElementsOnlyResolverTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Rule; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java index ab78297bcf9..912e6fcf030 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/NGramTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/NGramTestCase.java @@ -1,14 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.document.MatchType; -import com.yahoo.searchdefinition.document.Matching; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.Stemming; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.document.MatchType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.Stemming; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java index 4eeab15fdd2..a291dda24b9 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PagedAttributeValidatorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PagedAttributeValidatorTestCase.java @@ -1,15 +1,15 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.util.Optional; import static com.yahoo.config.model.test.TestUtil.joinLines; -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromStrings; +import static com.yahoo.schema.ApplicationBuilder.createFromString; +import static com.yahoo.schema.ApplicationBuilder.createFromStrings; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java index 6ddacd066b1..e5636da57a0 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ParentChildSearchModel.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ParentChildSearchModel.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.application.provider.MockFileRegistry; @@ -7,15 +7,14 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.DocumentReferences; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; -import com.yahoo.searchdefinition.document.TemporaryImportedField; -import com.yahoo.searchdefinition.document.TemporarySDField; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.DocumentReferences; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; +import com.yahoo.schema.document.TemporaryImportedField; +import com.yahoo.schema.document.TemporarySDField; /* * Fixture class used for ImportedFieldsResolverTestCase and AdjustPositionSummaryFieldsTestCase. diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java index a3af7706fb9..6f0facf9541 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/PositionTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/PositionTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.document.DataType; import com.yahoo.document.DocumentType; import com.yahoo.document.PositionDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.FieldSet; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.FieldSet; import com.yahoo.vespa.documentmodel.SummaryField; import com.yahoo.vespa.documentmodel.SummaryTransform; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java index 3a0ceebcb0a..69bf62be84b 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankModifierTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankModifierTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java b/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java index 2d0bdb58122..e380b1ab9af 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankProfileSearchFixture.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankProfileSearchFixture.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.google.common.collect.ImmutableList; import com.yahoo.config.application.api.ApplicationPackage; @@ -10,11 +10,11 @@ import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java index 0ab7406b5b1..dab1d9e6e95 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankPropertyVariablesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankPropertyVariablesTestCase.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile.RankProperty; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfile.RankProperty; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java index 9b32d97ef75..4b6a22fc81a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionTypeResolverTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionTypeResolverTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.TensorFieldType; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; import com.yahoo.searchlib.rankingexpression.rule.ReferenceNode; import com.yahoo.tensor.TensorType; import com.yahoo.yolean.Exceptions; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java index adaf1ef78f9..4df0a09ec2e 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithLightGBMTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithLightGBMTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.After; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java index 2057bcaba04..713e11fd608 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxModelTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.config.model.application.provider.FilesApplicationPackage; @@ -8,6 +8,7 @@ import com.yahoo.io.IOUtils; import com.yahoo.path.Path; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.vespa.config.search.core.OnnxModelsConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.search.DocumentDatabase; import com.yahoo.vespa.model.search.IndexedSearchCluster; @@ -57,6 +58,15 @@ public class RankingExpressionWithOnnxModelTestCase { private void assertGeneratedConfig(VespaModel vespaModel) { DocumentDatabase db = ((IndexedSearchCluster)vespaModel.getSearchClusters().get(0)).getDocumentDbs().get(0); + + RankingConstantsConfig.Builder rankingConstantsConfigBuilder = new RankingConstantsConfig.Builder(); + db.getConfig(rankingConstantsConfigBuilder); + var rankingConstantsConfig = rankingConstantsConfigBuilder.build(); + assertEquals(1, rankingConstantsConfig.constant().size()); + assertEquals("my_constant", rankingConstantsConfig.constant(0).name()); + assertEquals("tensor(d0[2])", rankingConstantsConfig.constant(0).type()); + assertEquals("files/constant.json", rankingConstantsConfig.constant(0).fileref().value()); + OnnxModelsConfig.Builder builder = new OnnxModelsConfig.Builder(); ((OnnxModelsConfig.Producer) db).getConfig(builder); OnnxModelsConfig config = new OnnxModelsConfig(builder); @@ -83,6 +93,18 @@ public class RankingExpressionWithOnnxModelTestCase { assertEquals("path_to_output_2", model.output(2).as()); model = config.model(1); + assertEquals("dynamic_model", model.name()); + assertEquals(1, model.input().size()); + assertEquals(1, model.output().size()); + assertEquals("rankingExpression(my_function)", model.input(0).source()); + + model = config.model(2); + assertEquals("unbound_model", model.name()); + assertEquals(1, model.input().size()); + assertEquals(1, model.output().size()); + assertEquals("rankingExpression(my_function)", model.input(0).source()); + + model = config.model(3); assertEquals("files_model_onnx", model.name()); assertEquals(3, model.input().size()); assertEquals(3, model.output().size()); @@ -94,27 +116,15 @@ public class RankingExpressionWithOnnxModelTestCase { assertEquals("path_to_output_2", model.output(2).as()); assertEquals("files_model_onnx", model.name()); - model = config.model(2); + model = config.model(4); assertEquals("another_model", model.name()); assertEquals("third_input", model.input(2).name()); assertEquals("rankingExpression(another_function)", model.input(2).source()); - model = config.model(3); + model = config.model(5); assertEquals("files_summary_model_onnx", model.name()); assertEquals(3, model.input().size()); assertEquals(3, model.output().size()); - - model = config.model(4); - assertEquals("unbound_model", model.name()); - assertEquals(1, model.input().size()); - assertEquals(1, model.output().size()); - assertEquals("rankingExpression(my_function)", model.input(0).source()); - - model = config.model(5); - assertEquals("dynamic_model", model.name()); - assertEquals(1, model.input().size()); - assertEquals(1, model.output().size()); - assertEquals("rankingExpression(my_function)", model.input(0).source()); } private void assertTransformedFeature(VespaModel model) { diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java index 7e562f825f4..94a51d25717 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithOnnxTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithOnnxTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationFile; import com.yahoo.config.application.api.ApplicationPackage; @@ -8,11 +8,9 @@ import com.yahoo.io.IOUtils; import com.yahoo.io.reader.NamedReader; import com.yahoo.path.Path; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.parser.ParseException; -import com.yahoo.searchlib.rankingexpression.evaluation.Value; +import com.yahoo.schema.FeatureNames; +import com.yahoo.schema.parser.ParseException; import com.yahoo.tensor.TensorType; -import com.yahoo.vespa.model.VespaModel; -import com.yahoo.vespa.model.ml.ImportedModelTester; import com.yahoo.yolean.Exceptions; import org.junit.After; import org.junit.Test; @@ -22,9 +20,8 @@ import java.io.FileReader; import java.io.IOException; import java.io.UncheckedIOException; import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Optional; +import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; @@ -234,7 +231,7 @@ public class RankingExpressionWithOnnxTestCase { search.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); assertNull("Constant overridden by function is not added", - search.search().rankingConstants().get( name + "_Variable")); + search.search().constants().get(name + "_Variable")); // At this point the expression is stored - copy application to another location which do not have a models dir Path storedApplicationDirectory = applicationDir.getParentPath().append("copy"); @@ -249,7 +246,7 @@ public class RankingExpressionWithOnnxTestCase { searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile"); searchFromStored.assertFirstPhaseExpression(vespaExpressionWithoutConstant, "my_profile_child"); assertNull("Constant overridden by function is not added", - searchFromStored.search().rankingConstants().get( name + "_Variable")); + searchFromStored.search().constants().get(name + "_Variable")); } finally { IOUtils.recursiveDeleteDir(storedApplicationDirectory.toFile()); } @@ -326,7 +323,7 @@ public class RankingExpressionWithOnnxTestCase { } private void assertSmallConstant(String name, TensorType type, RankProfileSearchFixture search) { - var value = search.compiledRankProfile("my_profile").getConstants().get(name); + var value = search.compiledRankProfile("my_profile").constants().get(FeatureNames.asConstantFeature(name)); assertNotNull(value); assertEquals(type, value.type()); } @@ -388,8 +385,8 @@ public class RankingExpressionWithOnnxTestCase { StoringApplicationPackage(Path applicationPackageWritableRoot, String queryProfile, String queryProfileType) { super(new File(applicationPackageWritableRoot.toString()), - null, null, Collections.emptyList(), null, - null, null, false, queryProfile, queryProfileType); + null, null, List.of(), Map.of(), null, + null, null, false, queryProfile, queryProfileType); } @Override diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java index 114486a5ddc..1f065bc7a20 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTensorTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTensorTestCase.java @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import static org.junit.Assert.assertEquals; @@ -13,7 +13,7 @@ import static org.junit.Assert.fail; public class RankingExpressionWithTensorTestCase { @Test - public void requireThatSingleLineConstantTensorAndTypeCanBeParsed() throws ParseException { + public void requireThatSingleLineConstantMappedTensorCanBeParsed() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + " first-phase {\n" + @@ -30,6 +30,40 @@ public class RankingExpressionWithTensorTestCase { } @Test + public void requireThatSingleLineConstantIndexedTensorCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):{ {x:0}:1, {x:1}:2, {x:2}:3 }\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test + public void requireThatSingleLineConstantIndexedTensorShortFormCanBeParsed() throws ParseException { + RankProfileSearchFixture f = new RankProfileSearchFixture( + " rank-profile my_profile {\n" + + " first-phase {\n" + + " expression: sum(my_tensor)\n" + + " }\n" + + " constants {\n" + + " my_tensor tensor(x[3]):[1, 2, 3]\n" + + " }\n" + + " }"); + f.compileRankProfile("my_profile"); + f.assertFirstPhaseExpression("reduce(constant(my_tensor), sum)", "my_profile"); + f.assertRankProperty("tensor(x[3]):[1.0, 2.0, 3.0]", "constant(my_tensor).value", "my_profile"); + f.assertRankProperty("tensor(x[3])", "constant(my_tensor).type", "my_profile"); + } + + @Test public void requireConstantTensorCanBeReferredViaConstantFeature() throws ParseException { RankProfileSearchFixture f = new RankProfileSearchFixture( " rank-profile my_profile {\n" + diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java index ffc073ed434..f8086fb3bc6 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithTransformerTokensTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithTransformerTokensTestCase.java @@ -1,18 +1,18 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.expressiontransforms.RankProfileTransformContext; -import com.yahoo.searchdefinition.expressiontransforms.TokenTransformer; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.expressiontransforms.RankProfileTransformContext; +import com.yahoo.schema.expressiontransforms.TokenTransformer; +import com.yahoo.schema.parser.ParseException; import com.yahoo.searchlib.rankingexpression.RankingExpression; import com.yahoo.searchlib.rankingexpression.evaluation.MapContext; import com.yahoo.searchlib.rankingexpression.evaluation.TensorValue; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXGBoostTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java index 0c163954038..e1b1473a59a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionWithXGBoostTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionWithXGBoostTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.parser.ParseException; import org.junit.After; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java index b42f932ef98..ace3788e49a 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/RankingExpressionsTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/RankingExpressionsTestCase.java @@ -1,22 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.collections.Pair; import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.search.query.profile.QueryProfileRegistry; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.derived.DerivedConfiguration; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java index 780e87b01f0..57b4d928a52 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReferenceFieldTestCase.java @@ -1,14 +1,14 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.document.DataType; import com.yahoo.document.Field; import com.yahoo.documentmodel.NewDocumentReferenceDataType; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.parser.ParseException; import org.junit.Rule; import org.junit.Test; import org.junit.rules.ExpectedException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java index 6d722138871..974d8c261ca 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedDocumentNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReservedDocumentNamesTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.derived.AbstractExportingTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.derived.AbstractExportingTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java index d3d79fd7798..e405a105f3c 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ReservedRankingExpressionFunctionNamesTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ReservedRankingExpressionFunctionNamesTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.DeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.util.logging.Level; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java b/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java index 4cfe4c09052..03f9d7c5960 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SchemaMustHaveDocumentTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SchemaMustHaveDocumentTest.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java index f224dc0688e..76132a4d09f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryConsistencyTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryConsistencyTestCase.java @@ -1,9 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.documentmodel.SummaryTransform; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java index 1eb5e66df93..d94815015d7 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/SummaryFieldsMustHaveValidSourceTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/SummaryFieldsMustHaveValidSourceTestCase.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.BaseDeployLogger; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.model.container.search.QueryProfiles; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java index b9e6bfe7dc9..9164f361a92 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorFieldTestCase.java @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; -import static com.yahoo.searchdefinition.ApplicationBuilder.createFromString; +import static com.yahoo.schema.ApplicationBuilder.createFromString; import static com.yahoo.config.model.test.TestUtil.joinLines; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -55,6 +55,17 @@ public class TensorFieldTestCase { } @Test + public void requireThatIndexedTensorAttributeCannotBeFastRank() throws ParseException { + try { + createFromString(getSd("field f1 type tensor(x[3]) { indexing: attribute \n attribute: fast-rank }")); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("The attribute 'f1' (tensor(x[3])) does not support 'fast-rank'. Only supported for tensor types with at least one mapped dimension", e.getMessage()); + } + } + + @Test public void requireThatIllegalTensorTypeSpecThrowsException() throws ParseException { try { createFromString(getSd("field f1 type tensor(invalid) { indexing: attribute }")); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java index 26b108e2b45..aaf5f381c62 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/TensorTransformTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/TensorTransformTestCase.java @@ -1,5 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.collections.Pair; import com.yahoo.component.ComponentId; @@ -10,15 +10,15 @@ import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; -import com.yahoo.searchdefinition.LargeRankExpressions; -import com.yahoo.searchdefinition.RankProfile; -import com.yahoo.searchdefinition.RankProfileRegistry; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.derived.AttributeFields; -import com.yahoo.searchdefinition.derived.RawRankProfile; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.LargeRankExpressions; +import com.yahoo.schema.RankProfile; +import com.yahoo.schema.RankProfileRegistry; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.derived.AttributeFields; +import com.yahoo.schema.derived.RawRankProfile; +import com.yahoo.schema.parser.ParseException; import ai.vespa.rankingexpression.importer.configmodelview.ImportedMlModels; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java index 8c801d9deaf..87bb2e96042 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/ValidateFieldTypesTest.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/ValidateFieldTypesTest.java @@ -1,20 +1,19 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.model.application.provider.MockFileRegistry; import com.yahoo.config.model.deploy.TestProperties; import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.document.DataType; import com.yahoo.document.Field; -import com.yahoo.searchdefinition.Application; -import com.yahoo.searchdefinition.DocumentReference; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; -import com.yahoo.searchdefinition.document.ImportedField; -import com.yahoo.searchdefinition.document.ImportedFields; -import com.yahoo.searchdefinition.document.ImportedSimpleField; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.DocumentReference; +import com.yahoo.schema.Schema; +import com.yahoo.schema.derived.TestableDeployLogger; +import com.yahoo.schema.document.ImportedField; +import com.yahoo.schema.document.ImportedFields; +import com.yahoo.schema.document.ImportedSimpleField; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.documentmodel.DocumentSummary; import com.yahoo.vespa.documentmodel.SummaryField; import org.junit.Rule; diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java index e0427d93ee4..016e30e80af 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/VespaMlModelTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/VespaMlModelTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; import com.yahoo.config.application.api.ApplicationPackage; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.derived.RawRankProfile; +import com.yahoo.schema.derived.RawRankProfile; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.ml.ImportedModelTester; import org.junit.After; @@ -68,7 +68,7 @@ public class VespaMlModelTestCase { private String rankConfigOf(String rankProfileName, VespaModel model) { StringBuilder b = new StringBuilder(); - RawRankProfile profile = model.rankProfileList().getRankProfile(rankProfileName); + RawRankProfile profile = model.rankProfileList().getRankProfiles().get(rankProfileName); for (var property : profile.configProperties()) b.append(property.getFirst()).append(" : ").append(property.getSecond()).append("\n"); return b.toString(); diff --git a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java b/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java index e56d6ccf343..2f62228cc3f 100644 --- a/config-model/src/test/java/com/yahoo/searchdefinition/processing/WeightedSetSummaryToTestCase.java +++ b/config-model/src/test/java/com/yahoo/schema/processing/WeightedSetSummaryToTestCase.java @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.searchdefinition.processing; +package com.yahoo.schema.processing; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java index c0d74cd02d3..768bf408250 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/AbstractReferenceFieldTestCase.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; +import com.yahoo.schema.AbstractSchemaTestCase; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java index 33d84cf4313..faf4b67430c 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderImportedFieldsTestCase.java @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.documentmodel; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java index ebb4156f3c0..3c2193ca4df 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderReferenceTypeTestCase.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.parser.ParseException; import org.junit.Test; import java.io.IOException; diff --git a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java index dc88590a198..d345c776db3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/documentmodel/DocumentModelBuilderTestCase.java @@ -3,9 +3,9 @@ package com.yahoo.vespa.documentmodel; import com.yahoo.document.config.DocumenttypesConfig; import com.yahoo.document.config.DocumentmanagerConfig; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.AbstractSchemaTestCase; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.AbstractSchemaTestCase; +import com.yahoo.schema.parser.ParseException; import com.yahoo.vespa.configmodel.producers.DocumentManager; import com.yahoo.vespa.configmodel.producers.DocumentTypes; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantValidatorTest.java new file mode 100644 index 00000000000..ff45038a051 --- /dev/null +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ConstantValidatorTest.java @@ -0,0 +1,34 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.model.application.validation; + +import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.fail; + +public class ConstantValidatorTest { + + @Test + public void ensure_that_valid_ranking_constants_do_not_fail() { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_ok/").create(); + } + + @Test + public void ensure_that_failing_ranking_constants_fails() { + try { + new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); + fail(); + } catch (IllegalArgumentException e) { + String[] lines = e.getMessage().split("\n"); + assertStartsWith("constant(constant_tensor_2) tensor(x[6]): file:tensors/constant_tensor_2.json: Tensor label is not a string", lines[1]); + assertStartsWith("constant(constant_tensor_3) tensor(cpp{},d{}): file:tensors/constant_tensor_3.json: Tensor dimension 'cd' does not exist", lines[2]); + assertStartsWith("constant(constant_tensor_4) tensor(x{},y{}): file:tensors/constant_tensor_4.json: Tensor dimension 'z' does not exist", lines[3]); + } + } + + private void assertStartsWith(String prefix, String value) { + assertEquals(prefix, value.substring(0, prefix.length())); + } + +} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java deleted file mode 100644 index f6fa73dce74..00000000000 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/RankingConstantsValidatorTest.java +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.model.application.validation; - -import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithFilePkg; -import org.junit.Test; - -import static com.yahoo.vespa.model.application.validation.RankingConstantsValidator.TensorValidationException; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -public class RankingConstantsValidatorTest { - - @Test - public void ensure_that_valid_ranking_constants_do_not_fail() { - new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_ok/").create(); - } - - @Test - public void ensure_that_failing_ranking_constants_fails() { - try { - new VespaModelCreatorWithFilePkg("src/test/cfg/application/validation/ranking_constants_fail/").create(); - fail(); - } catch (TensorValidationException e) { - assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_2' (tensors/constant_tensor_2.json): Tensor label is not a string (VALUE_NUMBER_INT)")); - assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_3' (tensors/constant_tensor_3.json): Tensor dimension 'cd' does not exist")); - assertTrue(e.getMessage().contains("Ranking constant 'constant_tensor_4' (tensors/constant_tensor_4.json): Tensor dimension 'z' does not exist")); - } - } - -} diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java index 666aef6df73..a7f3d134b57 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/ValidationTester.java @@ -23,8 +23,8 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.List; -import static com.yahoo.config.model.test.MockApplicationPackage.BOOK_SEARCHDEFINITION; -import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SEARCHDEFINITION; +import static com.yahoo.config.model.test.MockApplicationPackage.BOOK_SCHEMA; +import static com.yahoo.config.model.test.MockApplicationPackage.MUSIC_SCHEMA; /** * @author bratseth @@ -77,7 +77,7 @@ public class ValidationTester { Provisioned provisioned = hostProvisioner.startProvisionedRecording(); ApplicationPackage newApp = new MockApplicationPackage.Builder() .withServices(services) - .withSchemas(ImmutableList.of(MUSIC_SEARCHDEFINITION, BOOK_SEARCHDEFINITION)) + .withSchemas(ImmutableList.of(MUSIC_SCHEMA, BOOK_SCHEMA)) .withValidationOverrides(validationOverrides) .build(); VespaModelCreatorWithMockPkg newModelCreator = new VespaModelCreatorWithMockPkg(newApp); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java index fa6fc1244d8..567534ba9f6 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/ConfigValueChangeValidatorTest.java @@ -24,6 +24,7 @@ import org.junit.Test; import java.time.Instant; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java index 3cfde4c4d19..c3aa387da54 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/AttributeChangeValidatorTest.java @@ -76,6 +76,14 @@ public class AttributeChangeValidatorTest { } @Test + public void changing_fast_rank_require_restart() throws Exception { + new Fixture("field f1 type tensor(x{}) { indexing: attribute }", + "field f1 type tensor(x{}) { indexing: attribute \n attribute: fast-rank }"). + assertValidation(newRestartAction(ClusterSpec.Id.from("test"), + "Field 'f1' changed: add attribute 'fast-rank'")); + } + + @Test public void changing_btree2hash_require_restart() throws Exception { new Fixture("field f1 type long { indexing: attribute\n attribute: fast-search\n dictionary: btree}", "field f1 type long { indexing: attribute\n attribute: fast-search\n dictionary: hash }"). diff --git a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java index 86385b2f80d..859194fae79 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/application/validation/change/search/DocumentTypeChangeValidatorTest.java @@ -8,7 +8,7 @@ import com.yahoo.document.Field; import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.document.StructDataType; import com.yahoo.documentmodel.NewDocumentType; -import com.yahoo.searchdefinition.FieldSets; +import com.yahoo.schema.FieldSets; import com.yahoo.vespa.model.application.validation.change.VespaConfigChangeAction; import com.yahoo.vespa.model.application.validation.change.VespaRefeedAction; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index b05404ec08f..3468d81b748 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -32,7 +32,6 @@ import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; -import static com.yahoo.vespa.config.search.core.ProtonConfig.Feeding.Shared_field_writer_executor; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; @@ -770,7 +769,7 @@ public class ContentBuilderTest extends DomBuilderTest { DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(hostedXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); @@ -797,7 +796,7 @@ public class ContentBuilderTest extends DomBuilderTest { var deployStateBuilder = new DeployState.Builder().properties(props); var model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices(hostedXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); return getProtonConfig(model.getContentClusters().values().iterator().next()); @@ -866,7 +865,7 @@ public class ContentBuilderTest extends DomBuilderTest { return new MockApplicationPackage.Builder() .withHosts(hosts) .withServices(services) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build(); } @@ -887,7 +886,7 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); @@ -907,8 +906,8 @@ public class ContentBuilderTest extends DomBuilderTest { VespaModel m = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withHosts(getHosts()) .withServices(combined) - .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SEARCHDEFINITION, - MockApplicationPackage.BOOK_SEARCHDEFINITION)) + .withSchemas(Arrays.asList(MockApplicationPackage.MUSIC_SCHEMA, + MockApplicationPackage.BOOK_SCHEMA)) .build()) .create(); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java index f246b87d9bf..3deeef7f2a2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ml/ModelsEvaluatorTest.java @@ -18,8 +18,11 @@ import static org.junit.Assume.assumeTrue; public class ModelsEvaluatorTest { @Test - public void testModelsEvaluatorTester() { + public void testModelsEvaluator() { + // Assumption fails but test passes on Intel macs + // Assumption fails and test fails on ARM64 assumeTrue(OnnxEvaluator.isRuntimeAvailable()); + ModelsEvaluator modelsEvaluator = ModelsEvaluatorTester.create("src/test/cfg/application/stateless_eval"); assertEquals(3, modelsEvaluator.models().size()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java index 44ca69d55c8..d48c397b1f0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/ImplicitIndexingClusterTest.java @@ -54,7 +54,7 @@ public class ImplicitIndexingClusterTest { return new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() .withServices("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" + servicesXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .withSearchDefinition(MockApplicationPackage.MUSIC_SCHEMA) .build()) .create(deployStateBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 977c10d0828..0d51e4b6e74 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -2,7 +2,7 @@ package com.yahoo.vespa.model.container.search.searchchain; import com.yahoo.config.model.deploy.DeployState; -import com.yahoo.searchdefinition.derived.DerivedConfiguration; +import com.yahoo.schema.derived.DerivedConfiguration; import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.config.search.RankProfilesConfig; import com.yahoo.config.model.producer.AbstractConfigProducerRoot; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java index ab0b2f0d92c..e73eb905566 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/test/QueryProfilesTestCase.java @@ -8,14 +8,13 @@ import com.yahoo.search.query.profile.QueryProfile; import com.yahoo.search.query.profile.QueryProfileRegistry; import com.yahoo.search.query.profile.QueryProfileVariant; import com.yahoo.search.query.profile.compiled.CompiledQueryProfile; -import com.yahoo.search.query.profile.compiled.CompiledQueryProfileRegistry; import com.yahoo.search.query.profile.config.QueryProfileConfigurer; import com.yahoo.search.query.profile.config.QueryProfileXMLReader; import com.yahoo.search.query.profile.types.FieldDescription; import com.yahoo.search.query.profile.types.FieldType; import com.yahoo.search.query.profile.types.QueryProfileType; import com.yahoo.search.query.profile.types.QueryProfileTypeRegistry; -import com.yahoo.searchdefinition.derived.TestableDeployLogger; +import com.yahoo.schema.derived.TestableDeployLogger; import com.yahoo.vespa.model.container.search.QueryProfiles; import com.yahoo.vespa.model.test.utils.DeployLoggerStub; import org.junit.Test; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java index 16b4a8bed9b..81649375c8a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/ContainerModelBuilderTest.java @@ -1095,6 +1095,39 @@ public class ContainerModelBuilderTest extends ContainerModelBuilderTestBase { logger.msgs.get(1).getSecond()); } + @Test + public void logs_accesslog_not_overidable_in_hosted() { + String containerService = joinLines("<container id='foo' version='1.0'>", + " <accesslog type='json' fileNamePattern='logs/vespa/qrs/access.%Y%m%d%H%M%S' symlinkName='json_access' />", + " <nodes count=\"2\">", + " </nodes>", + "</container>"); + + String deploymentXml = joinLines("<deployment version='1.0'>", + " <prod>", + " <region>us-east-1</region>", + " </prod>", + "</deployment>"); + + ApplicationPackage applicationPackage = new MockApplicationPackage.Builder() + .withServices(containerService) + .withDeploymentSpec(deploymentXml) + .build(); + + TestLogger logger = new TestLogger(); + DeployState deployState = new DeployState.Builder() + .applicationPackage(applicationPackage) + .zone(new Zone(Environment.prod, RegionName.from("us-east-1"))) + .properties(new TestProperties().setHostedVespa(true)) + .deployLogger(logger) + .build(); + createModel(root, deployState, null, DomBuilderTest.parse(containerService)); + assertFalse(logger.msgs.isEmpty()); + assertEquals(Level.WARNING, logger.msgs.get(0).getFirst()); + assertEquals("Applications are not allowed to override the 'accesslog' element", + logger.msgs.get(0).getSecond()); + } + private void assertComponentConfigured(ApplicationContainerCluster cluster, String componentId) { Component<?, ?> component = cluster.getComponentsMap().get(ComponentId.fromString(componentId)); assertNotNull(component); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java index 0cff915266f..b2c29b88e38 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/xml/JvmOptionsTest.java @@ -148,24 +148,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatInvalidJvmGcOptionsAreLogged() throws IOException, SAXException { - verifyLoggingOfJvmGcOptions(true, - "-XX:+ParallelGCThreads=8 foo bar", - "foo", "bar"); - verifyLoggingOfJvmGcOptions(true, - "-XX:+UseCMSInitiatingOccupancyOnly foo bar", - "-XX:+UseCMSInitiatingOccupancyOnly", "foo", "bar"); - verifyLoggingOfJvmGcOptions(true, - "-XX:+UseConcMarkSweepGC", - "-XX:+UseConcMarkSweepGC"); - verifyLoggingOfJvmGcOptions(true, - "$(touch /tmp/hello-from-gc-options)", - "$(touch", "/tmp/hello-from-gc-options)"); - - verifyLoggingOfJvmGcOptions(false, - "$(touch /tmp/hello-from-gc-options)", - "$(touch", "/tmp/hello-from-gc-options)"); - + public void requireThatValidJvmGcOptionsAreNotLogged() throws IOException, SAXException { // Valid options, should not log anything verifyLoggingOfJvmGcOptions(true, "-XX:+ParallelGCThreads=8"); verifyLoggingOfJvmGcOptions(true, "-XX:MaxTenuringThreshold=15"); // No + or - after colon @@ -175,7 +158,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { @Test public void requireThatInvalidJvmGcOptionsFailDeployment() throws IOException, SAXException { try { - buildModelWithJvmOptions(new TestProperties().setHostedVespa(true).failDeploymentWithInvalidJvmOptions(true), + buildModelWithJvmOptions(new TestProperties().setHostedVespa(true), new TestLogger(), "gc-options", "-XX:+ParallelGCThreads=8 foo bar"); @@ -232,26 +215,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { } @Test - public void requireThatJvmOptionsAreLogged() throws IOException, SAXException { - verifyLoggingOfJvmOptions(true, - "options", - "-Xms2G foo bar", - "foo", "bar"); - verifyLoggingOfJvmOptions(true, - "options", - "$(touch /tmp/hello-from-gc-options)", - "$(touch", "/tmp/hello-from-gc-options)"); - - verifyLoggingOfJvmOptions(true, - "options", - "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005", - "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"); - - verifyLoggingOfJvmOptions(false, - "options", - "$(touch /tmp/hello-from-gc-options)", - "$(touch", "/tmp/hello-from-gc-options)"); - + public void requireThatValidJvmOptionsAreNotLogged() throws IOException, SAXException { // Valid options, should not log anything verifyLoggingOfJvmOptions(true, "options", "-Xms2G"); verifyLoggingOfJvmOptions(true, "options", "-Xlog:gc"); @@ -263,7 +227,7 @@ public class JvmOptionsTest extends ContainerModelBuilderTestBase { @Test public void requireThatInvalidJvmOptionsFailDeployment() throws IOException, SAXException { try { - buildModelWithJvmOptions(new TestProperties().setHostedVespa(true).failDeploymentWithInvalidJvmOptions(true), + buildModelWithJvmOptions(new TestProperties().setHostedVespa(true), new TestLogger(), "options", "-Xms2G foo bar"); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java index a1fc13adf5c..bd07513b704 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/ml/ImportedModelTester.java @@ -1,14 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.ml; -import com.google.common.collect.ImmutableList; +import com.yahoo.config.FileReference; import com.yahoo.config.model.ApplicationPackageTester; import ai.vespa.rankingexpression.importer.configmodelview.MlModelImporter; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.io.GrowableByteBuffer; import com.yahoo.io.IOUtils; import com.yahoo.path.Path; -import com.yahoo.searchdefinition.RankingConstant; import ai.vespa.rankingexpression.importer.lightgbm.LightGBMImporter; import ai.vespa.rankingexpression.importer.onnx.OnnxImporter; import ai.vespa.rankingexpression.importer.tensorflow.TensorFlowImporter; @@ -21,10 +20,12 @@ import org.xml.sax.SAXException; import java.io.IOException; import java.io.UncheckedIOException; +import java.util.List; import java.util.Optional; -import static junit.framework.TestCase.assertTrue; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; /** * Helper for testing of imported models. @@ -34,11 +35,11 @@ import static org.junit.Assert.assertEquals; */ public class ImportedModelTester { - private final ImmutableList<MlModelImporter> importers = ImmutableList.of(new TensorFlowImporter(), - new OnnxImporter(), - new LightGBMImporter(), - new XGBoostImporter(), - new VespaImporter()); + private final List<MlModelImporter> importers = List.of(new TensorFlowImporter(), + new OnnxImporter(), + new LightGBMImporter(), + new XGBoostImporter(), + new VespaImporter()); private final String modelName; private final Path applicationDir; @@ -69,9 +70,12 @@ public class ImportedModelTester { public void assertLargeConstant(String constantName, VespaModel model, Optional<Long> expectedSize) { try { Path constantApplicationPackagePath = Path.fromString("models.generated/" + modelName + "/constants").append(constantName + ".tbf"); - RankingConstant rankingConstant = model.rankingConstants().get(constantName); - assertEquals(constantName, rankingConstant.getName()); - assertTrue(rankingConstant.getFileName().endsWith(constantApplicationPackagePath.toString())); + var constant = model.rankProfileList().constants().asMap().get(constantName); + assertNotNull(constant); + assertEquals(constantName, constant.getName()); + assertTrue(constant.getFileName().endsWith(constantApplicationPackagePath.toString())); + + assertTrue(model.fileReferences().contains(new FileReference(constant.getFileName()))); if (expectedSize.isPresent()) { Path constantPath = applicationDir.append(constantApplicationPackagePath); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 671a9f6660d..d541a6422e7 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -4,12 +4,15 @@ package com.yahoo.vespa.model.search.test; import com.google.common.collect.ImmutableMap; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.deploy.TestProperties; -import com.yahoo.vespa.config.search.IndexschemaConfig; -import com.yahoo.vespa.config.search.core.ProtonConfig; -import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.IndexInfoConfig; import com.yahoo.vespa.config.search.AttributesConfig; +import com.yahoo.vespa.config.search.IndexschemaConfig; +import com.yahoo.vespa.config.search.RankProfilesConfig; +import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.config.search.core.RankingConstantsConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; @@ -17,10 +20,11 @@ import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; import org.junit.Test; -import java.util.List; import java.util.Arrays; -import java.util.Map; import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -38,21 +42,21 @@ public class DocumentDatabaseTestCase { @Test public void requireThatConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency("index", "", 0.50, 0.50); - verifyConcurrency("streaming", "", 1.0, 0.0); - verifyConcurrency("store-only", "", 1.0, 0.0); + verifyConcurrency("index", "", 0.50); + verifyConcurrency("streaming", "", 1.0); + verifyConcurrency("store-only", "", 1.0); } @Test public void requireThatFeatureFlagConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency("index", "", 0.30, 0.30, 0.3); - verifyConcurrency("streaming", "", 0.6, 0.0, 0.3); - verifyConcurrency("store-only", "", 0.8, 0.0, 0.4); + verifyConcurrency("index", "", 0.30, 0.3); + verifyConcurrency("streaming", "", 0.6, 0.3); + verifyConcurrency("store-only", "", 0.8, 0.4); } @Test public void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0, Arrays.asList(0.50, 0.0)); + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 1.0); } @Test @@ -60,7 +64,7 @@ public class DocumentDatabaseTestCase { String feedTuning = "<feeding>" + " <concurrency>0.7</concurrency>" + "</feeding>\n"; - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7, Arrays.asList(0.7, 0.0)); + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7); } @Test @@ -68,25 +72,24 @@ public class DocumentDatabaseTestCase { String feedTuning = "<feeding>" + " <concurrency>0.7</concurrency>" + "</feeding>\n"; - verifyConcurrency("index", feedTuning, 0.7, 0.7); - verifyConcurrency("streaming", feedTuning, 0.7, 0.0); - verifyConcurrency("store-only", feedTuning, 0.7, 0.0); + verifyConcurrency("index", feedTuning, 0.7); + verifyConcurrency("streaming", feedTuning, 0.7); + verifyConcurrency("store-only", feedTuning, 0.7); } - private void verifyConcurrency(String mode, String xmlTuning, double global, double local, double featureFlagConcurrency) { - verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, global, Arrays.asList(local), featureFlagConcurrency); + private void verifyConcurrency(String mode, String xmlTuning, double expectedConcurrency, double featureFlagConcurrency) { + verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, expectedConcurrency, featureFlagConcurrency); } - private void verifyConcurrency(String mode, String xmlTuning, double global, double local) { - verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, global, Arrays.asList(local), null); + private void verifyConcurrency(String mode, String xmlTuning, double expectedConcurrency) { + verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, expectedConcurrency, null); } - private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double global, List<Double> local) { - verifyConcurrency(nameAndModes, xmlTuning, global, local, null); + private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double expectedConcurrency) { + verifyConcurrency(nameAndModes, xmlTuning, expectedConcurrency, null); } - private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double global, List<Double> local, Double featureFlagConcurrency) { - assertEquals(nameAndModes.size(), local.size()); + private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double expectedConcurrency, Double featureFlagConcurrency) { TestProperties properties = new TestProperties(); if (featureFlagConcurrency != null) { properties.setFeedConcurrency(featureFlagConcurrency); @@ -94,12 +97,8 @@ public class DocumentDatabaseTestCase { var tester = new SchemaTester(); VespaModel model = tester.createModel(nameAndModes, xmlTuning, new DeployState.Builder().properties(properties)); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); - assertEquals(global, proton.feeding().concurrency(), SMALL); - assertEquals(local.size(), proton.documentdb().size()); - for (int i = 0; i < local.size(); i++) { - assertEquals(local.get(i), proton.documentdb(i).feeding().concurrency(), SMALL); - } + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); + assertEquals(expectedConcurrency, proton.feeding().concurrency(), SMALL); } @Test @@ -109,7 +108,7 @@ public class DocumentDatabaseTestCase { DocType.create("b", "streaming"), DocType.create("c", "store-only")), ""); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode()); assertEquals("a", proton.documentdb(0).inputdoctypename()); @@ -124,7 +123,7 @@ public class DocumentDatabaseTestCase { assertEquals(nameAndModes.size(), local.size()); VespaModel model = tester.createModel(nameAndModes, xmlTuning); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(local.size(), proton.documentdb().size()); for (int i = 0; i < local.size(); i++) { assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs()); @@ -171,7 +170,7 @@ public class DocumentDatabaseTestCase { String type3Id = "test/search/cluster.test/type3"; { assertEquals(3, indexedSearchCluster.getDocumentDbs().size()); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(3, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -210,6 +209,43 @@ public class DocumentDatabaseTestCase { } @Test + public void testRankingConstants() { + List<String> schemas = List.of("type1"); + var tester = new SchemaTester(); + + // Use lz4 endings to avoid having to provide file content to be validated + String schemaConstants = + " constant constant_1 {" + + " file: constants/my_constant_1.json.lz4" + + " type: tensor<float>(x{},y{})" + + " }" + + " constant constant_2 {" + + " file: constants/my_constant_2.json.lz4" + + " type: tensor(x[1000])" + + " }"; + + Map<Path, String> constants = new HashMap<>(); + constants.put(Path.fromString("constants/my_constant_1.json.lz4"), ""); + constants.put(Path.fromString("constants/my_constant_2.json.lz4"), ""); + var model = tester.createModel(schemaConstants, "", schemas, constants); + IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); + RankingConstantsConfig.Builder b = new RankingConstantsConfig.Builder(); + indexedSearchCluster.getDocumentDbs().get(0).getConfig(b); + RankingConstantsConfig config = b.build(); + assertEquals(2, config.constant().size()); + + var constant1Config = config.constant().get(0); + assertEquals("constant_1", constant1Config.name()); + assertEquals("constants/my_constant_1.json.lz4", constant1Config.fileref().value()); + assertEquals("tensor<float>(x{},y{})", constant1Config.type()); + + var constant2Config = config.constant().get(1); + assertEquals("constant_2", constant2Config.name()); + assertEquals("constants/my_constant_2.json.lz4", constant2Config.fileref().value()); + assertEquals("tensor(x[1000])", constant2Config.type()); + } + + @Test public void requireThatRelevantConfigIsAvailableForClusterSearcher() { String inputsProfile = " rank-profile inputs {" + @@ -300,7 +336,7 @@ public class DocumentDatabaseTestCase { var model = tester.createModelWithMode(mode, sds, builder); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); - ProtonConfig proton = tester.getProtonCfg(contentSearchCluster); + ProtonConfig proton = tester.getProtonConfig(contentSearchCluster); assertEquals(sds.size(), proton.documentdb().size()); for (int i = 0; i < sds.size(); i++) { assertEquals(sds.get(i), proton.documentdb(i).inputdoctypename()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java index 52923779602..1d15f973d21 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaClusterTest.java @@ -6,11 +6,11 @@ import com.yahoo.config.model.test.MockApplicationPackage; import com.yahoo.container.QrSearchersConfig; import com.yahoo.document.DataType; import com.yahoo.search.config.ClusterConfig; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.document.Attribute; -import com.yahoo.searchdefinition.document.SDDocumentType; -import com.yahoo.searchdefinition.document.SDField; +import com.yahoo.schema.Schema; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.document.Attribute; +import com.yahoo.schema.document.SDDocumentType; +import com.yahoo.schema.document.SDField; import com.yahoo.vespa.config.search.DispatchConfig; import com.yahoo.vespa.indexinglanguage.expressions.AttributeExpression; import com.yahoo.vespa.indexinglanguage.expressions.ScriptExpression; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java index cf2135aea5a..b8a14bc763b 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaInfoTestCase.java @@ -25,7 +25,8 @@ public class SchemaInfoTestCase { " query(myDouble1) double: 0.5" + " query(myDouble2) tensor()" + " query(myMap) tensor(key{}): { label1:1.0,\n \"label2\": 2.0, 'label3': 3.0 }" + - " query(myVector) tensor(x[3]):\n\n[1 ,2.0,3]" + + " query(myVector1) tensor(x[3]):\n\n[1 ,2.0,3]" + + " query(myVector2) tensor(x[3]):{{x:0}:1,{x: 1}: 2 , { x:2}:3.0 }" + " query(myMatrix) tensor(x[2],y[3]):[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]" + " query(myMixed1) tensor(key{},x[2]): { key1:[-1.0, 1.1], key2: [1,2]}" + " query(myMixed2) tensor(k1{},k2{},x[2]): { {k1:l1,k2:l1}:[-1.0, 1.1], {k1:l1,k2:l2}: [1,2]}" + @@ -58,18 +59,19 @@ public class SchemaInfoTestCase { tester.assertRankProfile(schema, 5, "rankfeatures", false, true); var schemaInfoProfile = tester.assertRankProfile(schema, 6, "inputs", false, false); - assertEquals(9, schemaInfoProfile.input().size()); + assertEquals(10, schemaInfoProfile.input().size()); var rankProfilesProfile = rankProfilesConfig.rankprofile().get(6); assertEquals("inputs", rankProfilesProfile.name()); assertInput("query(foo)", "tensor<float>(x[10])", null, 0, schemaInfoProfile, rankProfilesProfile); assertInput("query(bar)", "tensor(key{},x[1000])", null, 1, schemaInfoProfile, rankProfilesProfile); assertInput("query(myDouble1)", "tensor()", "0.5", 2, schemaInfoProfile, rankProfilesProfile); assertInput("query(myDouble2)", "tensor()", null, 3, schemaInfoProfile, rankProfilesProfile); - assertInput("query(myMap)", "tensor(key{})", "{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile); - assertInput("query(myVector)", "tensor(x[3])", "{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile); - assertInput("query(myMatrix)", "tensor(x[2],y[3])", "{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 6, schemaInfoProfile, rankProfilesProfile); - assertInput("query(myMixed1)", "tensor(key{},x[2])", "{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 7, schemaInfoProfile, rankProfilesProfile); - assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMap)", "tensor(key{})", "tensor(key{}):{{key:label1}:1.0, {key:label2}:2.0, {key:label3}:3.0}", 4, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myVector1)", "tensor(x[3])", "tensor(x[3]):{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 5, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myVector2)", "tensor(x[3])", "tensor(x[3]):{{x:0}:1.0, {x:1}:2.0, {x:2}:3.0}", 6, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMatrix)", "tensor(x[2],y[3])", "tensor(x[2],y[3]):{{x:0,y:0}:1.0, {x:0,y:1}:2.0, {x:0,y:2}:3.0, {x:1,y:0}:4.0, {x:1,y:1}:5.0, {x:1,y:2}:6.0}", 7, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed1)", "tensor(key{},x[2])", "tensor(key{},x[2]):{{key:key1,x:0}:-1.0, {key:key1,x:1}:1.1, {key:key2,x:0}:1.0, {key:key2,x:1}:2.0}", 8, schemaInfoProfile, rankProfilesProfile); + assertInput("query(myMixed2)", "tensor(k1{},k2{},x[2])", "tensor(k1{},k2{},x[2]):{{k1:l1,k2:l1,x:0}:-1.0, {k1:l1,k2:l1,x:1}:1.1, {k1:l1,k2:l2,x:0}:1.0, {k1:l1,k2:l2,x:1}:2.0}", 9, schemaInfoProfile, rankProfilesProfile); assertEquals(2, schema.summaryclass().size()); assertEquals("default", schema.summaryclass(0).name()); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java index 7a077c1a0aa..6b740816775 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaTester.java @@ -2,6 +2,8 @@ package com.yahoo.vespa.model.search.test; import com.yahoo.config.model.deploy.DeployState; +import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; import com.yahoo.prelude.fastsearch.DocumentdbInfoConfig; import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; @@ -14,6 +16,7 @@ import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import static org.junit.Assert.assertEquals; @@ -72,21 +75,22 @@ public class SchemaTester { return retval.toString(); } - ProtonConfig getProtonCfg(ContentSearchCluster cluster) { + ProtonConfig getProtonConfig(ContentSearchCluster cluster) { ProtonConfig.Builder pb = new ProtonConfig.Builder(); cluster.getConfig(pb); return new ProtonConfig(pb); } void assertSingleSD(String mode) { - List<String> sds = List.of("type1"); - VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(sds, mode), - generateSchemas("", sds)).create(); + List<String> schemas = List.of("type1"); + VespaModel model = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, mode), + generateSchemas("", "", schemas), + Map.of()).create(); IndexedSearchCluster indexedSearchCluster = (IndexedSearchCluster)model.getSearchClusters().get(0); ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); assertEquals(1, indexedSearchCluster.getDocumentDbs().size()); String type1Id = "test/search/cluster.test/type1"; - ProtonConfig proton = getProtonCfg(contentSearchCluster); + ProtonConfig proton = getProtonConfig(contentSearchCluster); assertEquals(1, proton.documentdb().size()); assertEquals("type1", proton.documentdb(0).inputdoctypename()); assertEquals(type1Id, proton.documentdb(0).configid()); @@ -95,13 +99,29 @@ public class SchemaTester { VespaModel createModel(List<String> schemas) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - generateSchemas("", schemas)).create(); + generateSchemas("", "", schemas), + Map.of()).create(); } VespaModel createModelWithRankProfile(String rankProfile, List<String> schemas) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, "index"), - generateSchemas(rankProfile, schemas)).create(); + generateSchemas("", rankProfile, schemas), + Map.of()).create(); + } + + VespaModel createModelWithSchemaContent(String schemaContent, List<String> schemas) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(schemaContent, "", schemas), + Map.of()).create(); + } + + VespaModel createModel(String schemaContent, String rankProfile, List<String> schemas, Map<Path, String> files) { + return new VespaModelCreatorWithMockPkg(vespaHosts, + createVespaServices(schemas, "index"), + generateSchemas(schemaContent, rankProfile, schemas), + files).create(); } VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { @@ -111,27 +131,29 @@ public class SchemaTester { VespaModel createModelWithMode(String mode, List<String> schemas) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, mode), - generateSchemas("", schemas)).create(); + generateSchemas("", "", schemas), + Map.of()).create(); } VespaModel createModelWithMode(String mode, List<String> schemas, DeployState.Builder builder) { return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServices(schemas, mode), - generateSchemas("", schemas)).create(builder); + generateSchemas("", "", schemas), + Map.of()).create(builder); } VespaModel createModel(List<DocType> nameAndModes, String xmlTuning, DeployState.Builder builder) { - List<String> sds = new ArrayList<>(nameAndModes.size()); - for (DocType nameAndMode : nameAndModes) { - sds.add(nameAndMode.getType()); - } + List<String> schemas = new ArrayList<>(nameAndModes.size()); + for (DocType nameAndMode : nameAndModes) + schemas.add(nameAndMode.getType()); var creator = new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), - generateSchemas("", sds)); + generateSchemas("", "", schemas), + Map.of()); return builder != null ? creator.create(builder) : creator.create(); } - public static String generateSchema(String name, String field1, String field2, String rankProfile) { + public static String generateSchema(String name, String field1, String field2, String schemaContent, String rankProfile) { return "schema " + name + " {" + " document " + name + " {" + " field " + field1 + " type string {\n" + @@ -146,6 +168,7 @@ public class SchemaTester { " indexing: attribute \n" + " }\n" + " }\n" + + schemaContent + " rank-profile staticrank inherits default {" + " first-phase { expression: attribute(" + field2 + ") }" + " }" + @@ -163,18 +186,18 @@ public class SchemaTester { "}"; } - public static List<String> generateSchemas(String rankProfile, String ... sdNames) { - return generateSchemas(rankProfile, Arrays.asList(sdNames)); + public static List<String> generateSchemas(String schemaContent, String rankProfile, String ... schemaNames) { + return generateSchemas(schemaContent, rankProfile, Arrays.asList(schemaNames)); } - public static List<String> generateSchemas(String rankProfile, List<String> sdNames) { - List<String> sds = new ArrayList<>(); + public static List<String> generateSchemas(String schemaContent, String rankProfile, List<String> schemaNames) { + List<String> schemas = new ArrayList<>(); int i = 0; - for (String sdName : sdNames) { - sds.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), rankProfile)); + for (String sdName : schemaNames) { + schemas.add(generateSchema(sdName, "f" + (i + 1), "f" + (i + 2), schemaContent, rankProfile)); i = i + 2; } - return sds; + return schemas; } DocumentdbInfoConfig.Documentdb.Rankprofile assertRankProfile(DocumentdbInfoConfig.Documentdb db, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index a347c0a9c43..cbd6ac4bea9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -26,7 +26,7 @@ import static org.junit.Assert.assertTrue; * * @author geirst */ -public class SchemaNodeTest { +public class SearchNodeTest { private void assertBaseDir(String expected, SearchNode node) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); @@ -49,13 +49,13 @@ public class SchemaNodeTest { } private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, - NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { + NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, boolean loadCodeAsHugePages) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - Optional.empty(), Optional.empty(), isHosted, 0.0); + Optional.empty(), Optional.empty(), isHosted, loadCodeAsHugePages, 0.0); } private static SearchNode createSearchNode(MockRoot root) { - return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true); + return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, false); } @Test @@ -68,7 +68,7 @@ public class SchemaNodeTest { @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); prepare(root, node, true); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @@ -76,7 +76,7 @@ public class SchemaNodeTest { @Test public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.getDeployState()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -85,13 +85,27 @@ public class SchemaNodeTest { @Test public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), false); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.getDeployState()); assertTrue(node.getPreShutdownCommand().isPresent()); assertTrue(node.getPreShutdownCommand().get().contains("vespa-proton-cmd " + node.getRpcPort() + " prepareRestart")); } + private void verifyCodePlacement(boolean hugePages) { + MockRoot root = new MockRoot(""); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, false, hugePages); + node.setHostResource(new HostResource(new Host(node, "mynbode2"))); + node.initService(root.getDeployState()); + assertEquals(hugePages, node.getStartupCommand().contains("VESPA_LOAD_CODE_AS_HUGEPAGES=")); + } + + @Test + public void requireThatCodePageTypeCanBeControlled() { + verifyCodePlacement(true); + verifyCodePlacement(false); + } + private MockRoot createRoot(ModelContext.Properties properties) { return new MockRoot("", new DeployState.Builder().properties(properties).build()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java index d777fefbbe5..2c70c7b2da5 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/test/utils/VespaModelCreatorWithMockPkg.java @@ -11,10 +11,12 @@ import com.yahoo.config.model.api.ValidationParameters.CheckRouting; import com.yahoo.config.model.application.provider.SchemaValidators; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.test.MockApplicationPackage; +import com.yahoo.path.Path; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.application.validation.Validation; import java.util.List; +import java.util.Map; /** * For testing purposes only. @@ -32,7 +34,15 @@ public class VespaModelCreatorWithMockPkg { } public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas) { - this(new MockApplicationPackage.Builder().withHosts(hosts).withServices(services).withSchemas(schemas).build()); + this(hosts, services, schemas, Map.of()); + } + + public VespaModelCreatorWithMockPkg(String hosts, String services, List<String> schemas, Map<Path, String> files) { + this(new MockApplicationPackage.Builder().withHosts(hosts) + .withServices(services) + .withSchemas(schemas) + .withFiles(files) + .build()); } public VespaModelCreatorWithMockPkg(ApplicationPackage appPkg) { diff --git a/config-provisioning/CMakeLists.txt b/config-provisioning/CMakeLists.txt index 63a6c3cb3e7..f5a64dcf646 100644 --- a/config-provisioning/CMakeLists.txt +++ b/config-provisioning/CMakeLists.txt @@ -1,3 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(config-provisioning) +install_jar(config-provisioning-jar-with-dependencies.jar) install_config_definitions() diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java index 958a37e1432..70e88418fb7 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/Capacity.java @@ -1,6 +1,9 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision; +import java.util.Objects; +import java.util.Optional; + /** * A capacity request. * @@ -11,14 +14,12 @@ public final class Capacity { /** Resources should stay between these values, inclusive */ private final ClusterResources min, max; - private final boolean required; - private final boolean canFail; - private final NodeType type; + private final Optional<CloudAccount> cloudAccount; - private Capacity(ClusterResources min, ClusterResources max, boolean required, boolean canFail, NodeType type) { + private Capacity(ClusterResources min, ClusterResources max, boolean required, boolean canFail, NodeType type, Optional<CloudAccount> cloudAccount) { validate(min); validate(max); if (max.smallerThan(min)) @@ -29,6 +30,7 @@ public final class Capacity { this.required = required; this.canFail = canFail; this.type = type; + this.cloudAccount = Objects.requireNonNull(cloudAccount); } private static void validate(ClusterResources resources) { @@ -58,8 +60,13 @@ public final class Capacity { */ public NodeType type() { return type; } + /** Returns the cloud account where this capacity is requested */ + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + public Capacity withLimits(ClusterResources min, ClusterResources max) { - return new Capacity(min, max, required, canFail, type); + return new Capacity(min, max, required, canFail, type, cloudAccount); } @Override @@ -82,8 +89,13 @@ public final class Capacity { return from(resources, required, canFail, NodeType.tenant); } + // TODO(mpolden): Remove when config models < 7.590 are gone public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail) { - return new Capacity(min, max, required, canFail, NodeType.tenant); + return from(min, max, required, canFail, Optional.empty()); + } + + public static Capacity from(ClusterResources min, ClusterResources max, boolean required, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new Capacity(min, max, required, canFail, NodeType.tenant, cloudAccount); } /** Creates this from a node type */ @@ -92,7 +104,7 @@ public final class Capacity { } private static Capacity from(ClusterResources resources, boolean required, boolean canFail, NodeType type) { - return new Capacity(resources, resources, required, canFail, type); + return new Capacity(resources, resources, required, canFail, type, Optional.empty()); } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java new file mode 100644 index 00000000000..3609ea4af88 --- /dev/null +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/CloudAccount.java @@ -0,0 +1,19 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.config.provision; + +import ai.vespa.validation.PatternedStringWrapper; + +import java.util.regex.Pattern; + +/** + * Identifies an account in a public cloud, such as AWS or GCP. + * + * @author mpolden + */ +public class CloudAccount extends PatternedStringWrapper<CloudAccount> { + + public CloudAccount(String value) { + super(value, Pattern.compile("^[0-9]{12}$"), "cloud account"); + } + +} diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/DockerImage.java b/config-provisioning/src/main/java/com/yahoo/config/provision/DockerImage.java index bbcf4b48c1d..b9693685cf6 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/DockerImage.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/DockerImage.java @@ -24,6 +24,9 @@ public class DockerImage { this.registry = Objects.requireNonNull(registry, "registry must be non-null"); this.repository = Objects.requireNonNull(repository, "repository must be non-null"); this.tag = Objects.requireNonNull(tag, "tag must be non-null"); + + if (tag.isPresent() && tag.get().isBlank()) + throw new IllegalArgumentException("Set tag cannot be empty"); } /** Returns the registry-part of this, i.e. the host/port of the registry. */ @@ -58,6 +61,8 @@ public class DockerImage { /** Returns a copy of this with registry set to given value */ public DockerImage withRegistry(String registry) { + if (registry.isBlank()) throw new IllegalArgumentException("Registry cannot be empty"); + if (registry.charAt(registry.length() - 1) == '/') throw new IllegalArgumentException("Registry cannot end with '/': " + registry); return new DockerImage(registry, repository, tag); } @@ -86,26 +91,21 @@ public class DockerImage { return Objects.hash(registry, repository, tag); } - public static DockerImage from(String registry, String repository) { - return new DockerImage(registry, repository, Optional.empty()); - } - public static DockerImage fromString(String s) { if (s.isEmpty()) return EMPTY; - int firstPathSeparator = s.indexOf('/'); - if (firstPathSeparator < 0) throw new IllegalArgumentException("Missing path separator in '" + s + "'"); + int repositoryStart = s.lastIndexOf('/', s.lastIndexOf('/') - 1); + if (repositoryStart < 0) throw new IllegalArgumentException("Expected to find at least 2 path segments in: " + s); - String registry = s.substring(0, firstPathSeparator); - String repository = s.substring(firstPathSeparator + 1); - if (repository.isEmpty()) throw new IllegalArgumentException("Repository must be non-empty in '" + s + "'"); + String registry = s.substring(0, repositoryStart); + String repository = s.substring(repositoryStart + 1); int tagStart = repository.indexOf(':'); - if (tagStart < 0) return new DockerImage(registry, repository, Optional.empty()); + Optional<String> tag = tagStart < 0 ? Optional.empty() : Optional.of(repository.substring(tagStart + 1)); - String tag = repository.substring(tagStart + 1); - repository = repository.substring(0, tagStart); - return new DockerImage(registry, repository, Optional.of(tag)); + if (tagStart >= 0) repository = repository.substring(0, tagStart); + if (repository.isEmpty()) throw new IllegalArgumentException("Repository must be non-empty in '" + s + "'"); + return new DockerImage(registry, repository, tag); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/DockerImageTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/DockerImageTest.java index 71fd453aa26..2e0de192f03 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/DockerImageTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/DockerImageTest.java @@ -8,6 +8,7 @@ import java.util.Map; import java.util.Optional; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.fail; /** @@ -22,7 +23,8 @@ public class DockerImageTest { "registry.example.com:9999/vespa/vespa:7.42", new DockerImage("registry.example.com:9999", "vespa/vespa", Optional.of("7.42")), "registry.example.com/vespa/vespa:7.42", new DockerImage("registry.example.com", "vespa/vespa", Optional.of("7.42")), "registry.example.com:9999/vespa/vespa", new DockerImage("registry.example.com:9999", "vespa/vespa", Optional.empty()), - "registry.example.com/vespa/vespa", new DockerImage("registry.example.com", "vespa/vespa", Optional.empty()) + "registry.example.com/vespa/vespa", new DockerImage("registry.example.com", "vespa/vespa", Optional.empty()), + "registry.example.com/project/repo/vespa/vespa", new DockerImage("registry.example.com/project/repo", "vespa/vespa", Optional.empty()) ); tests.forEach((value, expected) -> { DockerImage parsed = DockerImage.fromString(value); @@ -36,17 +38,26 @@ public class DockerImageTest { } @Test + public void registry_cannot_contain_slash() { + DockerImage image = DockerImage.fromString("registry.example.com/vespa/vespa"); + assertThrows(IllegalArgumentException.class, () -> image.withRegistry("")); + assertThrows(IllegalArgumentException.class, () -> image.withRegistry("my-registry/path/")); + } + + @Test public void parse_invalid() { List<String> tests = List.of( "registry.example.com", "registry.example.com/", + "registry.example.com/repository", + "registry.example.com/repository:", "foo", "foo:1.2.3" ); for (var value : tests) { try { DockerImage.fromString(value); - fail("Expected failure"); + fail("Expected failure for: " + value); } catch (IllegalArgumentException ignored) { } } diff --git a/config-proxy/CMakeLists.txt b/config-proxy/CMakeLists.txt index 22b9369a63c..9f2311ae51e 100644 --- a/config-proxy/CMakeLists.txt +++ b/config-proxy/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(config-proxy) +install_jar(config-proxy-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-config-ctl.sh vespa-config-ctl bin) vespa_install_script(src/main/sh/vespa-config-loadtester.sh vespa-config-loadtester bin) diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java index 9cd9a7c0981..5c0e9539f4f 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigPayloadApplier.java @@ -273,8 +273,14 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { // Need to convert url into actual file if 'url' type is used } else if (isUrlField(builder, methodName)) { - UrlReference url = resolveUrl(Utf8.toString(value.asUtf8())); - invokeSetter(builder, methodName, url); + String url = Utf8.toString(value.asUtf8()); + if (url == null || url.length() == 0) { + invokeSetter(builder, methodName, ""); + } else { + UrlReference urlref = resolveUrl(Utf8.toString(value.asUtf8())); + invokeSetter(builder, methodName, urlref); + } + } else { Object object = getValueFromInspector(value); @@ -294,7 +300,7 @@ public class ConfigPayloadApplier<T extends ConfigInstance.Builder> { private UrlReference resolveUrl(String url) { if (urlDownloader == null) { - throw new RuntimeException("Resolving url field failed due to missing URL downloader."); + return new UrlReference(url); // assuming config server - just return the actual url. } File file = urlDownloader.waitFor(new UrlReference(url), 60 * 60); return new UrlReference(file.getAbsolutePath()); diff --git a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java index 3d3a27da713..9ececb71c56 100644 --- a/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java +++ b/config/src/main/java/com/yahoo/vespa/config/ConfigTransformer.java @@ -28,7 +28,7 @@ public class ConfigTransformer<T extends ConfigInstance> { private final Class<T> clazz; private static volatile PathAcquirer pathAcquirer = new IdentityPathAcquirer(); - private static volatile UrlDownloader urlDownloader; + private static volatile UrlDownloader urlDownloader = null; /** * For internal use only * diff --git a/configd/src/apps/sentinel/CMakeLists.txt b/configd/src/apps/sentinel/CMakeLists.txt index de384e8a3cd..79d4af7b3a4 100644 --- a/configd/src/apps/sentinel/CMakeLists.txt +++ b/configd/src/apps/sentinel/CMakeLists.txt @@ -25,5 +25,5 @@ vespa_add_executable(configd_config-sentinel_app DEPENDS fnet configdefinitions - staging_vespalib + vespalib ) diff --git a/configd/src/apps/sentinel/env.h b/configd/src/apps/sentinel/env.h index 01531c425bf..ec3e201cf5e 100644 --- a/configd/src/apps/sentinel/env.h +++ b/configd/src/apps/sentinel/env.h @@ -8,7 +8,7 @@ #include "model-owner.h" #include "rpcserver.h" #include "state-api.h" -#include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/net/http/state_server.h> namespace config::sentinel { diff --git a/configd/src/apps/sentinel/manager.h b/configd/src/apps/sentinel/manager.h index 48c95fe7a3d..ac2e2a56785 100644 --- a/configd/src/apps/sentinel/manager.h +++ b/configd/src/apps/sentinel/manager.h @@ -8,7 +8,7 @@ #include "service.h" #include "state-api.h" #include <vespa/config-sentinel.h> -#include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/net/http/state_server.h> #include <sys/types.h> #include <sys/select.h> diff --git a/configd/src/apps/sentinel/state-api.h b/configd/src/apps/sentinel/state-api.h index 2cb2d381821..e00a0e0ba05 100644 --- a/configd/src/apps/sentinel/state-api.h +++ b/configd/src/apps/sentinel/state-api.h @@ -2,17 +2,15 @@ #pragma once -#include <vespa/vespalib/net/simple_health_producer.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/net/http/simple_health_producer.h> +#include <vespa/vespalib/net/http/simple_component_config_producer.h> #include <vespa/vespalib/metrics/simple_metrics.h> -namespace config { -namespace sentinel { +namespace config::sentinel { struct StateApi { vespalib::SimpleHealthProducer myHealth; vespalib::SimpleComponentConfigProducer myComponents; }; -} // namespace config::sentinel -} // namespace config +} diff --git a/configdefinitions/CMakeLists.txt b/configdefinitions/CMakeLists.txt index e032ab51445..c374e93904e 100644 --- a/configdefinitions/CMakeLists.txt +++ b/configdefinitions/CMakeLists.txt @@ -9,4 +9,4 @@ vespa_define_module( src/vespa ) -install_fat_java_artifact(configdefinitions) +install_jar(configdefinitions-jar-with-dependencies.jar) diff --git a/configserver-flags/CMakeLists.txt b/configserver-flags/CMakeLists.txt index 0eac9d119f4..5609a3ebeb1 100644 --- a/configserver-flags/CMakeLists.txt +++ b/configserver-flags/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(configserver-flags) +install_jar(configserver-flags-jar-with-dependencies.jar) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java index 6fac2f7f73a..b13d4542eba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/ApplicationRepository.java @@ -507,7 +507,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // ---------------- Application operations ---------------------------------------------------------------- /** - * Deletes an application + * Deletes an application and associated resources * * @return true if the application was found and deleted, false if it was not present * @throws RuntimeException if deleting the application fails. This method is exception safe. @@ -522,17 +522,21 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye .map(lock -> new ApplicationTransaction(lock, transaction)); try (var applicationLock = tenantApplications.lock(applicationId)) { Optional<Long> activeSession = tenantApplications.activeSessionOf(applicationId); - if (activeSession.isEmpty()) return false; - - try { - Session session = getRemoteSession(tenant, activeSession.get()); - transaction.add(tenant.getSessionRepository().createSetStatusTransaction(session, Session.Status.DELETE)); - } catch (NotFoundException e) { - log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); + CompletionWaiter waiter; + if (activeSession.isPresent()) { + try { + Session session = getRemoteSession(tenant, activeSession.get()); + transaction.add(tenant.getSessionRepository().createSetStatusTransaction(session, Session.Status.DELETE)); + } catch (NotFoundException e) { + log.log(Level.INFO, TenantRepository.logPre(applicationId) + "Active session exists, but has not been deleted properly. Trying to cleanup"); + } + waiter = tenantApplications.createRemoveApplicationWaiter(applicationId); + } else { + // If there's no active session, we still want to clean up any resources created in a failing prepare + waiter = new NoopCompletionWaiter(); } Curator curator = tenantRepository.getCurator(); - CompletionWaiter waiter = tenantApplications.createRemoveApplicationWaiter(applicationId); transaction.add(new ContainerEndpointsCache(tenant.getPath(), curator).delete(applicationId)); // TODO: Not unit tested // Delete any application roles transaction.add(new ApplicationRolesStore(curator, tenant.getPath()).delete(applicationId)); @@ -553,7 +557,7 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye // Wait for app being removed on other servers waiter.awaitCompletion(Duration.ofSeconds(30)); - return true; + return activeSession.isPresent(); } finally { applicationTransaction.ifPresent(ApplicationTransaction::close); } @@ -1210,4 +1214,14 @@ public class ApplicationRepository implements com.yahoo.config.provision.Deploye } + private static class NoopCompletionWaiter implements CompletionWaiter { + + @Override + public void awaitCompletion(Duration timeout) {} + + @Override + public void notifyCompletion() {} + + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 249e5931f08..c49e79f8698 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -19,6 +19,7 @@ import com.yahoo.config.model.api.Reindexing; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; @@ -186,6 +187,7 @@ public class ModelContextImpl implements ModelContext { private final int metricsproxyNumThreads; private final int availableProcessors; private final boolean containerDumpHeapOnShutdownTimeout; + private final boolean loadCodeAsHugePages; private final double containerShutdownTimeout; private final int maxUnCommittedMemory; private final boolean forwardIssuesAsErrors; @@ -193,7 +195,6 @@ public class ModelContextImpl implements ModelContext { private final boolean unorderedMergeChaining; private final boolean useV8GeoPositions; private final int maxCompactBuffers; - private final boolean failDeploymentWithInvalidJvmOptions; private final List<String> ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String mergeThrottlingPolicy; @@ -230,6 +231,7 @@ public class ModelContextImpl implements ModelContext { this.metricsproxyNumThreads = flagValue(source, appId, version, Flags.METRICSPROXY_NUM_THREADS); this.availableProcessors = flagValue(source, appId, version, Flags.AVAILABLE_PROCESSORS); this.containerDumpHeapOnShutdownTimeout = flagValue(source, appId, version, Flags.CONTAINER_DUMP_HEAP_ON_SHUTDOWN_TIMEOUT); + this.loadCodeAsHugePages = flagValue(source, appId, version, Flags.LOAD_CODE_AS_HUGEPAGES); this.containerShutdownTimeout = flagValue(source, appId, version, Flags.CONTAINER_SHUTDOWN_TIMEOUT); this.maxUnCommittedMemory = flagValue(source, appId, version, Flags.MAX_UNCOMMITTED_MEMORY); this.forwardIssuesAsErrors = flagValue(source, appId, version, PermanentFlags.FORWARD_ISSUES_AS_ERRORS); @@ -237,7 +239,6 @@ public class ModelContextImpl implements ModelContext { this.unorderedMergeChaining = flagValue(source, appId, version, Flags.UNORDERED_MERGE_CHAINING); this.useV8GeoPositions = flagValue(source, appId, version, Flags.USE_V8_GEO_POSITIONS); this.maxCompactBuffers = flagValue(source, appId, version, Flags.MAX_COMPACT_BUFFERS); - this.failDeploymentWithInvalidJvmOptions = flagValue(source, appId, version, Flags.FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS); this.ignoredHttpUserAgents = flagValue(source, appId, version, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, version, Flags.ENABLE_SERVER_OCSP_STAPLING); this.mergeThrottlingPolicy = flagValue(source, appId, version, Flags.MERGE_THROTTLING_POLICY); @@ -277,12 +278,12 @@ public class ModelContextImpl implements ModelContext { @Override public int availableProcessors() { return availableProcessors; } @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } + @Override public boolean loadCodeAsHugePages() { return loadCodeAsHugePages; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; } @Override public boolean ignoreThreadStackSizes() { return ignoreThreadStackSizes; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } - @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @@ -364,6 +365,7 @@ public class ModelContextImpl implements ModelContext { private final List<String> tlsCiphersOverride; private final List<String> zoneDnsSuffixes; private final List<String> environmentVariables; + private final Optional<CloudAccount> cloudAccount; public Properties(ApplicationId applicationId, Version nodeVespaVersion, @@ -378,7 +380,8 @@ public class ModelContextImpl implements ModelContext { Optional<Quota> maybeQuota, List<TenantSecretStore> tenantSecretStores, SecretStore secretStore, - List<X509Certificate> operatorCertificates) { + List<X509Certificate> operatorCertificates, + Optional<CloudAccount> cloudAccount) { this.featureFlags = new FeatureFlags(flagSource, applicationId, nodeVespaVersion); this.applicationId = applicationId; this.multitenant = configserverConfig.multitenant() || configserverConfig.hostedVespa() || Boolean.getBoolean("multitenant"); @@ -406,6 +409,7 @@ public class ModelContextImpl implements ModelContext { this.zoneDnsSuffixes = configserverConfig.zoneDnsSuffixes(); this.environmentVariables = PermanentFlags.ENVIRONMENT_VARIABLES.bindTo(flagSource) .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + this.cloudAccount = cloudAccount; } @Override public ModelContext.FeatureFlags featureFlags() { return featureFlags; } @@ -490,6 +494,11 @@ public class ModelContextImpl implements ModelContext { @Override public List<String> environmentVariables() { return environmentVariables; } + @Override + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java index b581688d571..468dda605c6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java @@ -11,7 +11,6 @@ import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.api.ModelFactory; import com.yahoo.config.model.api.Provisioned; import com.yahoo.config.model.application.provider.MockFileRegistry; -import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; @@ -160,7 +159,8 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> { zkClient.readQuota(), zkClient.readTenantSecretStores(), secretStore, - zkClient.readOperatorCertificates()); + zkClient.readOperatorCertificates(), + zkClient.readCloudAccount()); } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java index e75a01fb8dd..175b6f6457f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java @@ -1,35 +1,31 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config.server.session; -import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; -import com.yahoo.config.model.api.ApplicationRoles; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.Quota; +import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.security.X509CertificateUtils; -import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.config.server.TimeoutBudget; -import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.vespa.config.server.http.SessionHandler; +import com.yahoo.vespa.config.server.tenant.CloudAccountSerializer; import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer; import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer; import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; -import org.eclipse.jetty.util.ssl.X509; import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -58,6 +54,7 @@ public final class PrepareParams { static final String FORCE_PARAM_NAME = "force"; static final String WAIT_FOR_RESOURCES_IN_PREPARE = "waitForResourcesInPrepare"; static final String OPERATOR_CERTIFICATES = "operatorCertificates"; + static final String CLOUD_ACCOUNT = "cloudAccount"; private final ApplicationId applicationId; private final TimeoutBudget timeoutBudget; @@ -75,6 +72,7 @@ public final class PrepareParams { private final Optional<Quota> quota; private final List<TenantSecretStore> tenantSecretStores; private final List<X509Certificate> operatorCertificates; + private final Optional<CloudAccount> cloudAccount; private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors, boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion, @@ -82,7 +80,8 @@ public final class PrepareParams { Optional<EndpointCertificateMetadata> endpointCertificateMetadata, Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain, Optional<Quota> quota, List<TenantSecretStore> tenantSecretStores, - boolean force, boolean waitForResourcesInPrepare, List<X509Certificate> operatorCertificates) { + boolean force, boolean waitForResourcesInPrepare, List<X509Certificate> operatorCertificates, + Optional<CloudAccount> cloudAccount) { this.timeoutBudget = timeoutBudget; this.applicationId = Objects.requireNonNull(applicationId); this.ignoreValidationErrors = ignoreValidationErrors; @@ -99,6 +98,7 @@ public final class PrepareParams { this.force = force; this.waitForResourcesInPrepare = waitForResourcesInPrepare; this.operatorCertificates = operatorCertificates; + this.cloudAccount = Objects.requireNonNull(cloudAccount); } public static class Builder { @@ -116,10 +116,10 @@ public final class PrepareParams { private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty(); private Optional<DockerImage> dockerImageRepository = Optional.empty(); private Optional<AthenzDomain> athenzDomain = Optional.empty(); - private Optional<ApplicationRoles> applicationRoles = Optional.empty(); private Optional<Quota> quota = Optional.empty(); private List<TenantSecretStore> tenantSecretStores = List.of(); private List<X509Certificate> operatorCertificates = List.of(); + private Optional<CloudAccount> cloudAccount = Optional.empty(); public Builder() { } @@ -213,11 +213,6 @@ public final class PrepareParams { return this; } - public Builder applicationRoles(ApplicationRoles applicationRoles) { - this.applicationRoles = Optional.ofNullable(applicationRoles); - return this; - } - public Builder quota(Quota quota) { this.quota = Optional.ofNullable(quota); return this; @@ -252,18 +247,24 @@ public final class PrepareParams { return this; } - public Builder withOperatorCertificates(List<X509Certificate> operatorCertificates) { + public Builder operatorCertificates(List<X509Certificate> operatorCertificates) { this.operatorCertificates = List.copyOf(operatorCertificates); return this; } + public Builder cloudAccount(CloudAccount cloudAccount) { + this.cloudAccount = Optional.ofNullable(cloudAccount); + return this; + } + public PrepareParams build() { return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun, verbose, isBootstrap, vespaVersion, containerEndpoints, endpointCertificateMetadata, dockerImageRepository, athenzDomain, quota, tenantSecretStores, force, waitForResourcesInPrepare, - operatorCertificates); + operatorCertificates, cloudAccount); } + } public static PrepareParams fromHttpRequest(HttpRequest request, TenantName tenant, Duration barrierTimeout) { @@ -295,7 +296,7 @@ public final class PrepareParams { .timeoutBudget(SessionHandler.getTimeoutBudget(getTimeout(params, barrierTimeout))) .applicationId(createApplicationId(params, tenant)) .vespaVersion(SlimeUtils.optionalString(params.field(VESPA_VERSION_PARAM_NAME)).orElse(null)) - .containerEndpointList(deserialize(params.field(CONTAINER_ENDPOINTS_PARAM_NAME), ContainerEndpointSerializer::endpointListFromSlime, Collections.emptyList())) + .containerEndpointList(deserialize(params.field(CONTAINER_ENDPOINTS_PARAM_NAME), ContainerEndpointSerializer::endpointListFromSlime, List.of())) .endpointCertificateMetadata(deserialize(params.field(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME), EndpointCertificateMetadataSerializer::fromSlime)) .dockerImageRepository(SlimeUtils.optionalString(params.field(DOCKER_IMAGE_REPOSITORY)).orElse(null)) .athenzDomain(SlimeUtils.optionalString(params.field(ATHENZ_DOMAIN)).orElse(null)) @@ -303,13 +304,15 @@ public final class PrepareParams { .tenantSecretStores(deserialize(params.field(TENANT_SECRET_STORES_PARAM_NAME), TenantSecretStoreSerializer::listFromSlime, List.of())) .force(booleanValue(params, FORCE_PARAM_NAME)) .waitForResourcesInPrepare(booleanValue(params, WAIT_FOR_RESOURCES_IN_PREPARE)) - .withOperatorCertificates(deserialize(params.field(OPERATOR_CERTIFICATES), PrepareParams::readOperatorCertificates, Collections.emptyList())) + .operatorCertificates(deserialize(params.field(OPERATOR_CERTIFICATES), PrepareParams::readOperatorCertificates, List.of())) + .cloudAccount(deserialize(params.field(CLOUD_ACCOUNT), CloudAccountSerializer::fromSlime, null)) .build(); } private static <T> T deserialize(Inspector field, Function<Inspector, T> mapper) { return deserialize(field, mapper, null); } + private static <T> T deserialize(Inspector field, Function<Inspector, T> mapper, T defaultValue) { return field.valid() ? mapper.apply(field) @@ -318,13 +321,11 @@ public final class PrepareParams { private static boolean booleanValue(Inspector inspector, String fieldName) { Inspector field = inspector.field(fieldName); - return field.valid() - ? field.asBool() - : false; + return field.valid() && field.asBool(); } private static Duration getTimeout(Inspector params, Duration defaultTimeout) { - if(params.field("timeout").valid()) { + if (params.field("timeout").valid()) { return Duration.ofSeconds(params.field("timeout").asLong()); } else { return defaultTimeout; @@ -419,4 +420,9 @@ public final class PrepareParams { public List<X509Certificate> operatorCertificates() { return operatorCertificates; } + + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java index 6019f3ff438..4c1da9dc3af 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java @@ -20,10 +20,10 @@ import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Zone; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.lang.SettableOptional; import com.yahoo.net.HostName; import com.yahoo.path.Path; import com.yahoo.vespa.config.server.TimeoutBudget; @@ -204,7 +204,8 @@ public class SessionPreparer { params.quota(), params.tenantSecretStores(), secretStore, - params.operatorCertificates()); + params.operatorCertificates(), + params.cloudAccount()); this.fileRegistry = fileDistributionFactory.createFileRegistry(serverDbSessionDir); this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry, permanentApplicationPackage, @@ -281,7 +282,8 @@ public class SessionPreparer { athenzDomain, params.quota(), params.tenantSecretStores(), - params.operatorCertificates()); + params.operatorCertificates(), + params.cloudAccount()); checkTimeout("write state to zookeeper"); } @@ -321,7 +323,8 @@ public class SessionPreparer { Optional<AthenzDomain> athenzDomain, Optional<Quota> quota, List<TenantSecretStore> tenantSecretStores, - List<X509Certificate> operatorCertificates) { + List<X509Certificate> operatorCertificates, + Optional<CloudAccount> cloudAccount) { ZooKeeperDeployer zkDeployer = zooKeeperClient.createDeployer(deployLogger); try { zkDeployer.deploy(applicationPackage, fileRegistryMap, allocatedHosts); @@ -334,6 +337,7 @@ public class SessionPreparer { zooKeeperClient.writeQuota(quota); zooKeeperClient.writeTenantSecretStores(tenantSecretStores); zooKeeperClient.writeOperatorCertificates(operatorCertificates); + zooKeeperClient.writeCloudAccount(cloudAccount); } catch (RuntimeException | IOException e) { zkDeployer.cleanup(); throw new RuntimeException("Error preparing session", e); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java index 0b34bd95f8e..de8cbcb4066 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionZooKeeperClient.java @@ -12,6 +12,7 @@ import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.AllocatedHosts; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.TenantName; import com.yahoo.path.Path; @@ -23,6 +24,7 @@ import com.yahoo.vespa.config.server.deploy.ZooKeeperClient; import com.yahoo.vespa.config.server.deploy.ZooKeeperDeployer; import com.yahoo.vespa.config.server.filedistribution.AddFileInterface; import com.yahoo.vespa.config.server.filedistribution.MockFileManager; +import com.yahoo.vespa.config.server.tenant.CloudAccountSerializer; import com.yahoo.vespa.config.server.tenant.OperatorCertificateSerializer; import com.yahoo.vespa.config.server.tenant.TenantRepository; import com.yahoo.vespa.config.server.tenant.TenantSecretStoreSerializer; @@ -63,6 +65,7 @@ public class SessionZooKeeperClient { private static final String QUOTA_PATH = "quota"; private static final String TENANT_SECRET_STORES_PATH = "tenantSecretStores"; private static final String OPERATOR_CERTIFICATES_PATH = "operatorCertificates"; + private static final String CLOUD_ACCOUNT_PATH = "cloudAccount"; private final Curator curator; private final TenantName tenantName; @@ -208,6 +211,10 @@ public class SessionZooKeeperClient { return sessionPath.append(OPERATOR_CERTIFICATES_PATH); } + private Path cloudAccountPath() { + return sessionPath.append(CLOUD_ACCOUNT_PATH); + } + public void writeVespaVersion(Version version) { curator.set(versionPath(), Utf8.toBytes(version.toString())); } @@ -309,6 +316,17 @@ public class SessionZooKeeperClient { .orElse(List.of()); } + public void writeCloudAccount(Optional<CloudAccount> cloudAccount) { + if (cloudAccount.isPresent()) { + byte[] data = uncheck(() -> SlimeUtils.toJsonBytes(CloudAccountSerializer.toSlime(cloudAccount.get()))); + curator.set(cloudAccountPath(), data); + } + } + + public Optional<CloudAccount> readCloudAccount() { + return curator.getData(cloudAccountPath()).map(SlimeUtils::jsonToSlime).map(slime -> CloudAccountSerializer.fromSlime(slime.get())); + } + /** * Create necessary paths atomically for a new session. * diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/CloudAccountSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/CloudAccountSerializer.java new file mode 100644 index 00000000000..2478cf6baeb --- /dev/null +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/CloudAccountSerializer.java @@ -0,0 +1,29 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.provision.CloudAccount; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; + +/** + * @author mpolden + */ +public class CloudAccountSerializer { + + private static final String ID_FIELD = "id"; + + private CloudAccountSerializer() {} + + public static CloudAccount fromSlime(Inspector object) { + return new CloudAccount(object.field(ID_FIELD).asString()); + } + + public static Slime toSlime(CloudAccount account) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + root.setString(ID_FIELD, account.value()); + return slime; + } + +} diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java index 1ad0765a25f..17a0a2e3cab 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ApplicationRepositoryTest.java @@ -333,14 +333,16 @@ public class ApplicationRepositoryTest { assertTrue(applicationRepository.delete(applicationId())); assertTrue(applicationRepository.getActiveSession(applicationId()).isEmpty()); assertEquals(Optional.empty(), sessionRepository.getRemoteSession(sessionId).applicationSet()); - assertTrue(provisioner.removed()); + assertEquals(1, provisioner.removeCount()); assertEquals(tenant().getName(), provisioner.lastApplicationId().tenant()); assertEquals(applicationId(), provisioner.lastApplicationId()); assertTrue(curator.exists(sessionNode)); assertEquals(Session.Status.DELETE.name(), Utf8.toString(curator.getData(sessionNode.append("sessionState")).get())); assertTrue(sessionFile.exists()); + // Deleting a non-existent application still attempts to remove resources assertFalse(applicationRepository.delete(applicationId())); + assertEquals(2, provisioner.removeCount()); } { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java index f08da1e65c2..0ba3a6d883c 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockProvisioner.java @@ -23,7 +23,7 @@ import java.util.List; public class MockProvisioner implements Provisioner { private boolean activated = false; - private boolean removed = false; + private int removeCount = 0; private boolean restarted = false; private ApplicationId lastApplicationId; private Collection<HostSpec> lastHosts; @@ -70,7 +70,7 @@ public class MockProvisioner implements Provisioner { if (failureOnRemove) throw new IllegalStateException("Unable to remove " + transaction.application()); - removed = true; + removeCount++; lastApplicationId = transaction.application(); } @@ -94,8 +94,8 @@ public class MockProvisioner implements Provisioner { return activated; } - public boolean removed() { - return removed; + public int removeCount() { + return removeCount; } public boolean restarted() { diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index 41e6fe98441..8848beb3821 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -75,7 +75,8 @@ public class ModelContextImplTest { Optional.empty(), List.of(), new SecretStoreProvider().get(), - List.of()), + List.of(), + Optional.empty()), Optional.empty(), Optional.empty(), new Version(7), diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java index edcb9a0fd4b..66da009946e 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/http/v2/SessionPrepareHandlerTest.java @@ -30,6 +30,7 @@ import org.junit.Before; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; + import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; @@ -291,7 +292,7 @@ public class SessionPrepareHandlerTest extends SessionHandlerTest { @Test public void test_docker_image_repository() { long sessionId = createSession(applicationId()); - String dockerImageRepository = "foo.bar.com:4443/baz"; + String dockerImageRepository = "foo.bar.com:4443/baz/qux"; request(HttpRequest.Method.PUT, sessionId, Map.of("dockerImageRepository", dockerImageRepository, "applicationName", applicationId().application().value())); applicationRepository.activate(tenantRepository.getTenant(tenant), sessionId, timeoutBudget, false); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java index 46c31533598..e5b550fdc1a 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/PrepareParamsTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateMetadata; import com.yahoo.config.model.api.TenantSecretStore; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.TenantName; import com.yahoo.container.jdisc.HttpRequest; @@ -67,6 +68,7 @@ public class PrepareParamsTest { assertTrue(prepareParams.vespaVersion().isEmpty()); assertTrue(prepareParams.getTimeoutBudget().hasTimeLeft()); assertTrue(prepareParams.containerEndpoints().isEmpty()); + assertTrue(prepareParams.cloudAccount().isEmpty()); } @Test @@ -190,6 +192,13 @@ public class PrepareParamsTest { assertPrepareParamsEqual(prepareParams, prepareParamsJson); } + @Test + public void testCloudAccount() { + String json = "{\"cloudAccount\": {\"id\": \"012345678912\"}}"; + PrepareParams params = PrepareParams.fromJson(json.getBytes(StandardCharsets.UTF_8), TenantName.defaultName(), Duration.ZERO); + assertEquals(new CloudAccount("012345678912"), params.cloudAccount().get()); + } + private void assertPrepareParamsEqual(PrepareParams urlParams, PrepareParams jsonParams) { assertEquals(urlParams.ignoreValidationErrors(), jsonParams.ignoreValidationErrors()); assertEquals(urlParams.isDryRun(), jsonParams.isDryRun()); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java index 72d7311ed9a..78548dea195 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java @@ -9,11 +9,13 @@ import com.yahoo.config.application.api.FileRegistry; import com.yahoo.config.model.api.ApplicationClusterEndpoint; import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.model.api.EndpointCertificateSecrets; +import com.yahoo.config.model.api.ModelContext; import com.yahoo.config.model.application.provider.BaseDeployLogger; import com.yahoo.config.model.application.provider.FilesApplicationPackage; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.CertificateNotReadyException; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; @@ -341,6 +343,25 @@ public class SessionPreparerTest { prepare(new File("src/test/resources/deploy/hosted-app"), params); } + @Test + public void require_that_cloud_account_is_written() throws Exception { + TestModelFactory modelFactory = new TestModelFactory(version123); + preparer = createPreparer(new ModelFactoryRegistry(List.of(modelFactory)), HostProvisionerProvider.empty()); + ApplicationId applicationId = applicationId("test"); + CloudAccount expected = new CloudAccount("012345678912"); + PrepareParams params = new PrepareParams.Builder().applicationId(applicationId) + .cloudAccount(expected) + .build(); + prepare(new File("src/test/resources/deploy/hosted-app"), params); + + SessionZooKeeperClient zkClient = createSessionZooKeeperClient(); + assertEquals(expected, zkClient.readCloudAccount().get()); + + ModelContext modelContext = modelFactory.getModelContext(); + Optional<CloudAccount> accountFromModel = modelContext.properties().cloudAccount(); + assertEquals(Optional.of(expected), accountFromModel); + } + private List<ContainerEndpoint> readContainerEndpoints(ApplicationId applicationId) { Path tenantPath = TenantRepository.getTenantPath(applicationId.tenant()); return new ContainerEndpointsCache(tenantPath, curator).read(applicationId); diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/CloudAccountSerializerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/CloudAccountSerializerTest.java new file mode 100644 index 00000000000..7989cdcf019 --- /dev/null +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/CloudAccountSerializerTest.java @@ -0,0 +1,22 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.config.server.tenant; + +import com.yahoo.config.provision.CloudAccount; +import org.junit.Test; + +import static com.yahoo.vespa.config.server.tenant.CloudAccountSerializer.fromSlime; +import static com.yahoo.vespa.config.server.tenant.CloudAccountSerializer.toSlime; +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class CloudAccountSerializerTest { + + @Test + public void serialization() { + CloudAccount account = new CloudAccount("012345678912"); + assertEquals(account, fromSlime(toSlime(account).get())); + } + +} diff --git a/container-apache-http-client-bundle/CMakeLists.txt b/container-apache-http-client-bundle/CMakeLists.txt index 861fda3395f..0f65b3308fc 100644 --- a/container-apache-http-client-bundle/CMakeLists.txt +++ b/container-apache-http-client-bundle/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(container-apache-http-client-bundle) +install_jar(container-apache-http-client-bundle-jar-with-dependencies.jar) diff --git a/container-core/CMakeLists.txt b/container-core/CMakeLists.txt index 5015fd22c0d..cb51d76b7e2 100644 --- a/container-core/CMakeLists.txt +++ b/container-core/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. install_config_definitions() -install_java_artifact_dependencies(container-core) +install_jar_dependencies(container-core) vespa_install_script(src/main/sh/find-pid libexec/vespa) vespa_install_script(src/main/sh/vespa-load-balancer-status libexec/vespa) diff --git a/container-core/pom.xml b/container-core/pom.xml index be22e5cae5c..4b47cfde122 100644 --- a/container-core/pom.xml +++ b/container-core/pom.xml @@ -287,12 +287,6 @@ <scope>test</scope> </dependency> <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-client</artifactId> - <version>${jetty.version}</version> - <scope>test</scope> - </dependency> - <dependency> <groupId>org.hamcrest</groupId> <artifactId>hamcrest-library</artifactId> <scope>test</scope> diff --git a/container-core/src/main/java/com/yahoo/container/Container.java b/container-core/src/main/java/com/yahoo/container/Container.java index 3d3bf221528..47731761630 100755 --- a/container-core/src/main/java/com/yahoo/container/Container.java +++ b/container-core/src/main/java/com/yahoo/container/Container.java @@ -24,6 +24,7 @@ import java.util.logging.Logger; public class Container { private volatile boolean usingCustomFileAcquirer = false; + private volatile boolean disabledUrlDownloader = false; private volatile ComponentRegistry<RequestHandler> requestHandlerRegistry; private volatile ComponentRegistry<ClientProvider> clientProviderRegistry; @@ -132,8 +133,15 @@ public class Container { } public void setupUrlDownloader() { + if (disabledUrlDownloader) { + return; + } this.urlDownloader = new UrlDownloader(); ConfigTransformer.setUrlDownloader(urlDownloader); } + public void disableUrlDownloader() { + disabledUrlDownloader = true; + } + } diff --git a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java index 95494190734..6b510aadd3f 100644 --- a/container-core/src/main/java/com/yahoo/container/handler/Coverage.java +++ b/container-core/src/main/java/com/yahoo/container/handler/Coverage.java @@ -63,9 +63,8 @@ public class Coverage { } public void merge(Coverage other) { - if (other == null) { - return; - } + if (other == null) return; + docs += other.getDocs(); nodes += other.getNodes(); nodesTried += other.nodesTried; @@ -77,17 +76,17 @@ public class Coverage { // explicitly incomplete beats doc count beats explicitly full switch (other.fullReason) { - case EXPLICITLY_FULL: - // do nothing - break; - case EXPLICITLY_INCOMPLETE: - fullReason = FullCoverageDefinition.EXPLICITLY_INCOMPLETE; - break; - case DOCUMENT_COUNT: - if (fullReason == FullCoverageDefinition.EXPLICITLY_FULL) { - fullReason = FullCoverageDefinition.DOCUMENT_COUNT; - } - break; + case EXPLICITLY_FULL: + // do nothing + break; + case EXPLICITLY_INCOMPLETE: + fullReason = FullCoverageDefinition.EXPLICITLY_INCOMPLETE; + break; + case DOCUMENT_COUNT: + if (fullReason == FullCoverageDefinition.EXPLICITLY_FULL) { + fullReason = FullCoverageDefinition.DOCUMENT_COUNT; + } + break; } } diff --git a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java index a572f0be998..e1acaa6bc67 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java +++ b/container-core/src/main/java/com/yahoo/container/logging/ConnectionLogHandler.java @@ -9,13 +9,15 @@ class ConnectionLogHandler { private final LogFileHandler<ConnectionLogEntry> logFileHandler; public ConnectionLogHandler(String logDirectoryName, int bufferSize, String clusterName, - int queueSize, LogWriter<ConnectionLogEntry> logWriter) { + int queueSize, LogWriter<ConnectionLogEntry> logWriter, boolean useClusterIdInFileName) { logFileHandler = new LogFileHandler<>( LogFileHandler.Compression.ZSTD, bufferSize, - String.format("logs/vespa/%s/ConnectionLog.%s.%s", logDirectoryName, clusterName, "%Y%m%d%H%M%S"), + useClusterIdInFileName ? String.format("logs/vespa/%s/ConnectionLog.%s.%s", logDirectoryName, clusterName, "%Y%m%d%H%M%S") : + String.format("logs/vespa/%s/ConnectionLog.%s", logDirectoryName, "%Y%m%d%H%M%S"), "0 60 ...", - String.format("ConnectionLog.%s", clusterName), + useClusterIdInFileName ? String.format("ConnectionLog.%s", clusterName) : + "ConnectionLog", queueSize, "connection-logger", logWriter); diff --git a/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java b/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java index 72c93443d31..57b4783153d 100644 --- a/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java +++ b/container-core/src/main/java/com/yahoo/container/logging/FileConnectionLog.java @@ -15,7 +15,7 @@ public class FileConnectionLog extends AbstractComponent implements ConnectionLo @Inject public FileConnectionLog(ConnectionLogConfig config) { logHandler = new ConnectionLogHandler(config.logDirectoryName(), config.bufferSize(), config.cluster(), - queueSize(config), new JsonConnectionLogWriter()); + queueSize(config), new JsonConnectionLogWriter(), config.useClusterIdInFileName()); } private static int queueSize(ConnectionLogConfig config) { diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java index 2535d057e0e..a7c5b83f6a6 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/ConnectorFactory.java @@ -13,6 +13,7 @@ import org.eclipse.jetty.http2.server.AbstractHTTP2ServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory; import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory; import org.eclipse.jetty.server.ConnectionFactory; +import org.eclipse.jetty.server.Connector; import org.eclipse.jetty.server.DetectorConnectionFactory; import org.eclipse.jetty.server.HttpConfiguration; import org.eclipse.jetty.server.HttpConnectionFactory; @@ -122,10 +123,7 @@ public class ConnectorFactory { sslFactory = newSslConnectionFactory(metric, http1Factory); } if (proxyProtocolConfig.enabled()) { - if (proxyProtocolConfig.mixedMode()) { - factories.add(newDetectorConnectionFactory(sslFactory)); - } - factories.add(newProxyProtocolConnectionFactory(sslFactory)); + factories.add(newProxyProtocolConnectionFactory(sslFactory, proxyProtocolConfig.mixedMode())); } factories.add(sslFactory); if (connectorConfig.http2Enabled()) factories.add(alpnFactory); @@ -199,8 +197,10 @@ public class ConnectorFactory { return new DetectorConnectionFactory(alternatives); } - private ProxyConnectionFactory newProxyProtocolConnectionFactory(ConnectionFactory wrappedFactory) { - return new ProxyConnectionFactory(wrappedFactory.getProtocol()); + private ProxyConnectionFactory newProxyProtocolConnectionFactory(ConnectionFactory wrapped, boolean mixedMode) { + return mixedMode + ? new ProxyConnectionFactory(wrapped.getProtocol()) + : new MandatoryProxyConnectionFactory(wrapped.getProtocol()); } private static boolean isSslEffectivelyEnabled(ConnectorConfig config) { @@ -210,4 +210,14 @@ public class ConnectorFactory { private static long toMillis(double seconds) { return (long)(seconds * 1000); } + /** + * A {@link ProxyConnectionFactory} which disables the default behaviour of upgrading to + * next protocol when proxy protocol is not detected. + */ + private static class MandatoryProxyConnectionFactory extends ProxyConnectionFactory { + MandatoryProxyConnectionFactory(String next) { super(next); } + @Override protected String findNextProtocol(Connector __) { return null; } + } + + } diff --git a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java index 5f55bcfe0b4..97ea30b3867 100644 --- a/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java +++ b/container-core/src/main/java/com/yahoo/jdisc/http/server/jetty/HealthCheckProxyHandler.java @@ -7,15 +7,11 @@ import com.yahoo.security.SslContextBuilder; import com.yahoo.security.tls.TransportSecurityOptions; import com.yahoo.security.tls.TransportSecurityUtils; import com.yahoo.security.tls.TrustAllX509TrustManager; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.util.EntityUtils; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory; +import org.eclipse.jetty.client.api.ContentResponse; +import org.eclipse.jetty.http.HttpField; +import org.eclipse.jetty.http.HttpHeader; import org.eclipse.jetty.server.DetectorConnectionFactory; import org.eclipse.jetty.server.Request; import org.eclipse.jetty.server.SslConnectionFactory; @@ -86,7 +82,9 @@ class HealthCheckProxyHandler extends HandlerWrapper { .map(detectorConnFactory -> detectorConnFactory.getBean(SslConnectionFactory.class))) .map(connFactory -> (SslContextFactory.Server) connFactory.getSslContextFactory()) .orElseThrow(() -> new IllegalArgumentException("Health check proxy can only target https port")); - return new ProxyTarget(targetPort, targetTimeout, sslContextFactory); + ConnectorConfig.ProxyProtocol proxyProtocolCfg = targetConnector.connectorConfig().proxyProtocol(); + boolean proxyProtocol = proxyProtocolCfg.enabled() && !proxyProtocolCfg.mixedMode(); + return new ProxyTarget(targetPort, targetTimeout, sslContextFactory, proxyProtocol); } @Override @@ -161,14 +159,16 @@ class HealthCheckProxyHandler extends HandlerWrapper { private static class ProxyTarget implements AutoCloseable { final int port; final Duration timeout; - final SslContextFactory.Server sslContextFactory; - volatile CloseableHttpClient client; + final SslContextFactory.Server serverSsl; + final boolean proxyProtocol; + volatile HttpClient client; volatile StatusResponse lastResponse; - ProxyTarget(int port, Duration timeout, SslContextFactory.Server sslContextFactory) { + ProxyTarget(int port, Duration timeout, SslContextFactory.Server serverSsl, boolean proxyProtocol) { this.port = port; this.timeout = timeout; - this.sslContextFactory = sslContextFactory; + this.serverSsl = serverSsl; + this.proxyProtocol = proxyProtocol; } StatusResponse requestStatusHtml() { @@ -180,16 +180,17 @@ class HealthCheckProxyHandler extends HandlerWrapper { } private StatusResponse getStatusResponse() { - try (CloseableHttpResponse clientResponse = client().execute(new HttpGet("https://localhost:" + port + HEALTH_CHECK_PATH))) { - int statusCode = clientResponse.getStatusLine().getStatusCode(); - HttpEntity entity = clientResponse.getEntity(); - if (entity != null) { - Header contentTypeHeader = entity.getContentType(); - String contentType = contentTypeHeader != null ? contentTypeHeader.getValue() : null; - byte[] content = EntityUtils.toByteArray(entity); - return new StatusResponse(statusCode, contentType, content); + try { + var request = client().newRequest("https://localhost:" + port + HEALTH_CHECK_PATH); + if (proxyProtocol) { + request.tag(new ProxyProtocolClientConnectionFactory.V1.Tag()); + } + ContentResponse response = request.send(); + byte[] content = response.getContent(); + if (content != null && content.length > 0) { + return new StatusResponse(response.getStatus(), response.getMediaType(), content); } else { - return new StatusResponse(statusCode, null, null); + return new StatusResponse(response.getStatus(), null, null); } } catch (Exception e) { log.log(Level.FINE, e, () -> "Proxy request failed" + e.getMessage()); @@ -198,26 +199,23 @@ class HealthCheckProxyHandler extends HandlerWrapper { } // Client construction must be delayed to ensure that the SslContextFactory is started before calling getSslContext(). - private CloseableHttpClient client() { + private HttpClient client() throws Exception { if (client == null) { synchronized (this) { if (client == null) { int timeoutMillis = (int) timeout.toMillis(); - client = HttpClientBuilder.create() - .disableAutomaticRetries() - .setMaxConnPerRoute(4) - .setSSLContext(getSslContext(sslContextFactory)) - .setSSLHostnameVerifier(NoopHostnameVerifier.INSTANCE) // Certificate may not match "localhost" - .setUserTokenHandler(context -> null) // https://stackoverflow.com/a/42112034/1615280 - .setUserAgent("health-check-proxy-client") - .setDefaultRequestConfig( - RequestConfig.custom() - .setConnectTimeout(timeoutMillis) - .setConnectionRequestTimeout(timeoutMillis) - .setSocketTimeout(timeoutMillis) - .build()) - .build(); - } + SslContextFactory.Client clientSsl = new SslContextFactory.Client(); + clientSsl.setHostnameVerifier((__, ___) -> true); + clientSsl.setSslContext(getSslContext(serverSsl)); + HttpClient client = new HttpClient(clientSsl); + client.setMaxConnectionsPerDestination(4); + client.setConnectTimeout(timeoutMillis); + client.setStopTimeout(timeoutMillis); + client.setIdleTimeout(timeoutMillis); + client.setUserAgentField(new HttpField(HttpHeader.USER_AGENT, "health-check-proxy-client")); + client.start(); + this.client = client; + } } } return client; @@ -247,10 +245,11 @@ class HealthCheckProxyHandler extends HandlerWrapper { } @Override - public void close() throws IOException { + public void close() throws Exception { synchronized (this) { if (client != null) { - client.close(); + client.stop(); + client.destroy(); client = null; } } diff --git a/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def b/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def index 026e2a85a8d..0d58684538a 100644 --- a/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def +++ b/container-core/src/main/resources/configdefinitions/container.logging.connection-log.def @@ -12,3 +12,6 @@ queueSize int default=10000 # Buffer size for the output stream has a default of 256k bufferSize int default=262144 + +# In hosted Vespa we do not use the clusterId in file names for application containers +useClusterIdInFileName bool default=true
\ No newline at end of file diff --git a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java index b862624efc0..d4d6dcee957 100644 --- a/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java +++ b/container-core/src/test/java/com/yahoo/jdisc/http/server/jetty/ProxyProtocolTest.java @@ -31,8 +31,10 @@ import static com.yahoo.yolean.Exceptions.uncheckInterrupted; import static org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V1; import static org.eclipse.jetty.client.ProxyProtocolClientConnectionFactory.V2; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author bjorncs @@ -117,6 +119,20 @@ class ProxyProtocolTest { assertNotEquals(proxyLocalPort, connectionLog.logEntries().get(0).localPort().get().intValue()); } + @Test + void requireThatSslConnectionFailsWhenMixedModeIsDisabled() throws Exception { + JettyTestDriver driver = createSslWithProxyProtocolTestDriver( + certificateFile, privateKeyFile, requestLogMock, connectionLog, false); + try { + sendJettyClientRequest(driver, certificateFile, null); + fail("Expected exception"); + } catch (ExecutionException e) { + assertInstanceOf(IOException.class, e.getCause()); + } finally { + assertTrue(driver.close()); + } + } + private static JettyTestDriver createSslWithProxyProtocolTestDriver( Path certificateFile, Path privateKeyFile, RequestLog requestLog, ConnectionLog connectionLog, boolean mixedMode) { @@ -142,6 +158,7 @@ class ProxyProtocolTest { private ContentResponse sendJettyClientRequest(JettyTestDriver testDriver, Path certificateFile, Object tag) throws Exception { HttpClient client = createJettyHttpClient(certificateFile); + ExecutionException cause = null; try { int maxAttempts = 3; for (int attempt = 0; attempt < maxAttempts; attempt++) { @@ -152,13 +169,14 @@ class ProxyProtocolTest { assertEquals(200, response.getStatus()); return response; } catch (ExecutionException e) { - // Retry when the server closes the connection before the TLS handshake is completed. This have been observed in CI. + // Retry when the server closes the connection before the TLS handshake is completed. This has been observed in CI. // We have been unable to reproduce this locally. The cause is therefor currently unknown. log.log(Level.WARNING, String.format("Attempt %d failed: %s", attempt, e.getMessage()), e); Thread.sleep(10); + cause = e; } } - throw new AssertionError("Failed to send request, see log for details"); + throw cause; } finally { client.stop(); client.destroy(); diff --git a/container-disc/CMakeLists.txt b/container-disc/CMakeLists.txt index c6f745c7d57..6b905c27a8c 100644 --- a/container-disc/CMakeLists.txt +++ b/container-disc/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(container-disc) +install_jar(container-disc-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-start-container-daemon.sh vespa-start-container-daemon bin) diff --git a/container-disc/pom.xml b/container-disc/pom.xml index f84e1868724..a94225f6b5e 100644 --- a/container-disc/pom.xml +++ b/container-disc/pom.xml @@ -227,6 +227,7 @@ http2-hpack-${jetty.version}.jar, jetty-alpn-java-server-${jetty.version}.jar, jetty-alpn-server-${jetty.version}.jar, + jetty-client-${jetty.version}.jar, jetty-continuation-${jetty.version}.jar, jetty-http-${jetty.version}.jar, jetty-io-${jetty.version}.jar, diff --git a/container-search-and-docproc/CMakeLists.txt b/container-search-and-docproc/CMakeLists.txt index 2a4a0718129..006bf92f230 100644 --- a/container-search-and-docproc/CMakeLists.txt +++ b/container-search-and-docproc/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(container-search-and-docproc) +install_jar(container-search-and-docproc-jar-with-dependencies.jar) install_config_definitions() diff --git a/container-search-gui/CMakeLists.txt b/container-search-gui/CMakeLists.txt index 4360e7ed01a..572a337c0d0 100644 --- a/container-search-gui/CMakeLists.txt +++ b/container-search-gui/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(container-search-gui)
\ No newline at end of file +install_jar(container-search-gui-jar-with-dependencies.jar)
\ No newline at end of file diff --git a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java index ab29f98898a..70f95998dcb 100644 --- a/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/prelude/cluster/ClusterSearcher.java @@ -18,7 +18,6 @@ import com.yahoo.search.Query; import com.yahoo.search.Result; import com.yahoo.search.Searcher; import com.yahoo.search.config.ClusterConfig; -import com.yahoo.search.config.SchemaInfoConfig; import com.yahoo.search.dispatch.Dispatcher; import com.yahoo.search.query.ParameterParser; import com.yahoo.search.result.ErrorMessage; @@ -30,11 +29,9 @@ import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.ExecutionException; @@ -63,9 +60,6 @@ public class ClusterSearcher extends Searcher { // The set of document types contained in this search cluster private final Set<String> schemas; - // Mapping from rank profile names to schemas containing them - private final Map<String, Set<String>> rankProfilesz = new HashMap<>(); - private final long maxQueryTimeout; // in milliseconds private final long maxQueryCacheTimeout; // in milliseconds diff --git a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java index 097d714b47b..71758666b99 100644 --- a/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java +++ b/container-search/src/main/java/com/yahoo/search/cluster/ClusterSearcher.java @@ -90,8 +90,8 @@ public abstract class ClusterSearcher<T> extends PingableSearcher implements Nod } catch (ExecutionException e) { pong = new Pong(ErrorMessage.createUnspecifiedError("Execution was interrupted: " + p)); logThrowable = e; - } catch (LinkageError e) { // Typically Osgi woes - pong = new Pong(ErrorMessage.createErrorInPluginSearcher("Class loading problem",e)); + } catch (LinkageError e) { // Typically, Osgi woes + pong = new Pong(ErrorMessage.createErrorInPluginSearcher("Class loading problem", e)); logThrowable = e; } catch (TimeoutException e) { pong = new Pong(ErrorMessage.createNoAnswerWhenPingingNode("Ping thread timed out.")); diff --git a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java index 32706d2f2c1..8e99f4948ce 100644 --- a/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java +++ b/container-search/src/main/java/com/yahoo/search/dispatch/rpc/RpcPing.java @@ -67,7 +67,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { var ping = SearchProtocol.MonitorRequest.newBuilder().build().toByteArray(); double timeoutSeconds = ((double) clusterMonitor.getConfiguration().getRequestTimeout()) / 1000.0; Compressor.Compression compressionResult = resourcePool.compressor().compress(PING_COMPRESSION, ping); - connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(),this, timeoutSeconds); + connection.request(RPC_METHOD, compressionResult.type(), ping.length, compressionResult.data(), this, timeoutSeconds); } private Pong decodeReply(ProtobufResponse response) throws InvalidProtocolBufferException { @@ -91,7 +91,7 @@ public class RpcPing implements Pinger, Client.ResponseReceiver { if (node.isLastReceivedPong(pingSequenceId)) { pongHandler.handle(toPong(response)); } else { - //TODO Reduce to debug or remove once we have enumerated what happens here. + // TODO: Reduce to debug or remove once we have enumerated what happens here. log.info("Pong " + pingSequenceId + " from node " + node.key() + " in group " + node.group() + " with hostname " + node.hostname() + " received too late, latest is " + node.getLastReceivedPongId()); } diff --git a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java index 1910078058a..e942d5dbdef 100644 --- a/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java +++ b/container-search/src/main/java/com/yahoo/search/query/profile/QueryProfileRegistry.java @@ -49,30 +49,30 @@ public class QueryProfileRegistry extends ComponentRegistry<QueryProfile> { * which has a type which allows path mahting is used. If there is no such profile, null is returned. */ public QueryProfile findQueryProfile(String idString) { - if (idString==null) return getComponent("default"); - ComponentSpecification id=new ComponentSpecification(idString); - QueryProfile profile=getComponent(id); - if (profile!=null) return profile; + if (idString == null) return getComponent("default"); + ComponentSpecification id = new ComponentSpecification(idString); + QueryProfile profile = getComponent(id); + if (profile != null) return profile; return findPathParentQueryProfile(new ComponentSpecification(idString)); } private QueryProfile findPathParentQueryProfile(ComponentSpecification id) { // Try the name with "/" appended - should have the same semantics with path matching - QueryProfile slashedProfile=getComponent(new ComponentSpecification(id.getName() + "/",id.getVersionSpecification())); - if (slashedProfile!=null && slashedProfile.getType()!=null && slashedProfile.getType().getMatchAsPath()) + QueryProfile slashedProfile = getComponent(new ComponentSpecification(id.getName() + "/", id.getVersionSpecification())); + if (slashedProfile != null && slashedProfile.getType() != null && slashedProfile.getType().getMatchAsPath()) return slashedProfile; // Extract the parent (if any) - int slashIndex=id.getName().lastIndexOf("/"); - if (slashIndex<1) return null; - String parentName=id.getName().substring(0,slashIndex); + int slashIndex = id.getName().lastIndexOf("/"); + if (slashIndex < 1) return null; + String parentName = id.getName().substring(0,slashIndex); - ComponentSpecification parentId=new ComponentSpecification(parentName,id.getVersionSpecification()); + ComponentSpecification parentId = new ComponentSpecification(parentName, id.getVersionSpecification()); - QueryProfile pathParentProfile=getComponent(parentId); + QueryProfile pathParentProfile = getComponent(parentId); - if (pathParentProfile!=null && pathParentProfile.getType()!=null && pathParentProfile.getType().getMatchAsPath()) + if (pathParentProfile != null && pathParentProfile.getType() != null && pathParentProfile.getType().getMatchAsPath()) return pathParentProfile; return findPathParentQueryProfile(parentId); } diff --git a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java index aaab86f248c..4e92c3696fd 100644 --- a/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java +++ b/container-search/src/main/java/com/yahoo/search/query/ranking/RankFeatures.java @@ -50,7 +50,10 @@ public class RankFeatures implements Cloneable { /** Sets a tensor rank feature */ public void put(String name, Tensor value) { verifyType(name, value); - features.put(name, value); + if (value.type().rank() == 0) + features.put(name, value.asDouble()); + else + features.put(name, value); } private void verifyType(String name, Object value) { diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java index ad98197fa93..a35d01f6891 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretSto import java.security.cert.X509Certificate; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -36,21 +38,9 @@ public class DeploymentData { private final Quota quota; private final List<TenantSecretStore> tenantSecretStores; private final List<X509Certificate> operatorCertificates; + private final Optional<CloudAccount> cloudAccount; private final boolean dryRun; - // TODO: Remove when users have been updated to use constructor below - public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, - Set<ContainerEndpoint> containerEndpoints, - Optional<EndpointCertificateMetadata> endpointCertificateMetadata, - Optional<DockerImage> dockerImageRepo, - Optional<AthenzDomain> athenzDomain, - Quota quota, - List<TenantSecretStore> tenantSecretStores, - List<X509Certificate> operatorCertificates) { - this(instance, zone, applicationPackage, platform, containerEndpoints, endpointCertificateMetadata, - dockerImageRepo, athenzDomain, quota, tenantSecretStores, operatorCertificates, false); - } - public DeploymentData(ApplicationId instance, ZoneId zone, byte[] applicationPackage, Version platform, Set<ContainerEndpoint> containerEndpoints, Optional<EndpointCertificateMetadata> endpointCertificateMetadata, @@ -59,7 +49,7 @@ public class DeploymentData { Quota quota, List<TenantSecretStore> tenantSecretStores, List<X509Certificate> operatorCertificates, - boolean dryRun) { + Optional<CloudAccount> cloudAccount, boolean dryRun) { this.instance = requireNonNull(instance); this.zone = requireNonNull(zone); this.applicationPackage = requireNonNull(applicationPackage); @@ -71,6 +61,7 @@ public class DeploymentData { this.quota = quota; this.tenantSecretStores = List.copyOf(requireNonNull(tenantSecretStores)); this.operatorCertificates = List.copyOf(requireNonNull(operatorCertificates)); + this.cloudAccount = Objects.requireNonNull(cloudAccount); this.dryRun = dryRun; } @@ -118,6 +109,10 @@ public class DeploymentData { return operatorCertificates; } + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + public boolean isDryRun() { return dryRun; } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ControllerIdentityProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ControllerIdentityProvider.java new file mode 100644 index 00000000000..d2a7fb01ae2 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/ControllerIdentityProvider.java @@ -0,0 +1,16 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration; + +import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; + +import javax.net.ssl.SSLSocketFactory; + +/** + * @author freva + */ +public interface ControllerIdentityProvider extends ServiceIdentityProvider { + + /** Returns SSLSocketFactory that creates appropriate sockets to talk to the different config servers */ + SSLSocketFactory getConfigServerSslSocketFactory(); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java index 0415b33b29d..53e2592e0a6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/athenz/ZmsClientMock.java @@ -17,6 +17,7 @@ import com.yahoo.vespa.athenz.client.zms.ZmsClient; import com.yahoo.vespa.athenz.client.zms.ZmsClientException; import com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId; +import java.security.PublicKey; import java.time.Instant; import java.util.ArrayList; import java.util.HashSet; @@ -224,6 +225,11 @@ public class ZmsClientMock implements ZmsClient { } @Override + public void updateServicePublicKey(AthenzService athenzService, String publicKeyId, PublicKey publicKey) { + + } + + @Override public void deleteService(AthenzService athenzService) { athenz.getOrCreateDomain(athenzService.getDomain()).services.remove(athenzService.getName()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 6907747646e..e48ad7596ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.TenantName; @@ -19,7 +20,6 @@ import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzPrincipal; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.ListFlag; @@ -41,7 +41,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.RestartFilter; @@ -60,9 +59,7 @@ import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.JobStatus; -import com.yahoo.vespa.hosted.controller.deployment.RevisionHistory; import com.yahoo.vespa.hosted.controller.deployment.Run; -import com.yahoo.vespa.hosted.controller.deployment.RunStatus; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -85,7 +82,6 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -93,7 +89,6 @@ import java.util.Optional; import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.Predicate; @@ -620,12 +615,14 @@ public class ApplicationController { List<X509Certificate> operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() .map(SupportAccessGrant::certificate) .collect(toList()); - + Optional<CloudAccount> cloudAccount = applicationPackage.deploymentSpec() + .instance(application.instance()) + .flatMap(spec -> spec.cloudAccount(zone.environment(), zone.region())); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, - dryRun)); + cloudAccount, dryRun)); return new ActivateResult(new com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java index 950eaea904a..ccad4fe92ad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java @@ -6,11 +6,16 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.Endpoint; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.ListFlag; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -24,6 +29,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.stream.Collectors; /** @@ -34,9 +40,11 @@ import java.util.stream.Collectors; public class ApplicationPackageValidator { private final Controller controller; + private final ListFlag<String> cloudAccountsFlag; public ApplicationPackageValidator(Controller controller) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.cloudAccountsFlag = PermanentFlags.CLOUD_ACCOUNTS.bindTo(controller.flagSource()); } /** @@ -46,6 +54,7 @@ public class ApplicationPackageValidator { */ public void validate(Application application, ApplicationPackage applicationPackage, Instant instant) { validateSteps(applicationPackage.deploymentSpec()); + validateCloudAccounts(application, applicationPackage.deploymentSpec()); validateEndpointRegions(applicationPackage.deploymentSpec()); validateEndpointChange(application, applicationPackage, instant); validateCompactedEndpoint(applicationPackage); @@ -145,6 +154,25 @@ public class ApplicationPackageValidator { ". " + ValidationOverrides.toAllowMessage(validationId)); } + /** Verify that declared cloud accounts are allowed to be used by the tenant */ + private void validateCloudAccounts(Application application, DeploymentSpec deploymentSpec) { + TenantName tenant = application.id().tenant(); + Set<CloudAccount> validAccounts = cloudAccountsFlag.with(FetchVector.Dimension.TENANT_ID, tenant.value()) + .value().stream() + .map(CloudAccount::new) + .collect(Collectors.toSet()); + for (var spec : deploymentSpec.instances()) { + for (var zone : spec.zones()) { + if (!zone.environment().isProduction()) continue; + Optional<CloudAccount> cloudAccount = spec.cloudAccount(zone.environment(), zone.region().get()); + if (cloudAccount.isEmpty()) continue; + if (validAccounts.contains(cloudAccount.get())) continue; + throw new IllegalArgumentException("Cloud account '" + cloudAccount.get().value() + + "' is not valid for tenant '" + tenant + "'"); + } + } + } + /** Returns whether newEndpoints contains all destinations in endpoints */ private static boolean containsAllDestinationsOf(List<Endpoint> endpoints, List<Endpoint> newEndpoints) { var containsAllRegions = true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index 19fb1b7e1bb..6d011438b10 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.auditlog; -import com.google.common.base.CharMatcher; import com.google.common.collect.Ordering; import java.time.Instant; @@ -10,7 +9,6 @@ import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.function.Predicate; /** * This represents the audit log of a hosted Vespa system. The audit log contains manual actions performed through @@ -66,7 +64,7 @@ public class AuditLog { private final String resource; private final Optional<String> data; - public Entry(Instant at, String principal, Method method, String resource, Optional<String> data) { + public Entry(Instant at, String principal, Method method, String resource, byte[] data) { this.at = Objects.requireNonNull(at, "at must be non-null"); this.principal = Objects.requireNonNull(principal, "principal must be non-null"); this.method = Objects.requireNonNull(method, "method must be non-null"); @@ -112,16 +110,27 @@ public class AuditLog { DELETE } - private static Optional<String> sanitize(Optional<String> data) { - Objects.requireNonNull(data, "data must be non-null"); - return data.filter(Predicate.not(String::isBlank)) - .filter(CharMatcher.ascii()::matchesAllOf) - .map(v -> { - if (v.length() > maxDataLength) { - return v.substring(0, maxDataLength); - } - return v; - }); + private static Optional<String> sanitize(byte[] data) { + StringBuilder sb = new StringBuilder(); + for (byte b : data) { + char c = (char) b; + if (!printableAscii(c) && !tabOrLineBreak(c)) { + return Optional.empty(); + } + sb.append(c); + if (sb.length() == maxDataLength) { + break; + } + } + return Optional.of(sb.toString()).filter(s -> !s.isEmpty()); + } + + private static boolean printableAscii(char c) { + return c >= 32 && c <= 126; + } + + private static boolean tabOrLineBreak(char c) { + return c == 9 || c == 10 || c == 13; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java index 34e7955e02a..b6782767386 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLogger.java @@ -3,14 +3,12 @@ package com.yahoo.vespa.hosted.controller.auditlog; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.security.Principal; import java.time.Clock; import java.time.Duration; @@ -70,7 +68,7 @@ public class AuditLogger { Instant now = clock.instant(); AuditLog.Entry entry = new AuditLog.Entry(now, principal.getName(), method.get(), pathAndQueryOf(request.getUri()), - Optional.of(new String(data, StandardCharsets.UTF_8))); + data); try (Mutex lock = db.lockAuditLog()) { AuditLog auditLog = db.readAuditLog() .pruneBefore(now.minus(entryTtl)) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 52e5431b552..fb7a3fc4ba5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -13,11 +13,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; -import com.yahoo.security.KeyAlgorithm; -import com.yahoo.security.KeyUtils; -import com.yahoo.security.SignatureAlgorithm; -import com.yahoo.security.X509CertificateBuilder; -import com.yahoo.security.X509CertificateUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; @@ -48,12 +43,9 @@ import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.yolean.Exceptions; -import javax.security.auth.x500.X500Principal; import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.io.UncheckedIOException; -import java.math.BigInteger; -import java.security.KeyPair; import java.security.cert.CertificateExpiredException; import java.security.cert.CertificateNotYetValidException; import java.security.cert.X509Certificate; @@ -65,7 +57,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.UUID; import java.util.function.Consumer; import java.util.function.Supplier; import java.util.logging.Level; @@ -95,7 +86,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.deployReal; import static com.yahoo.vespa.hosted.controller.deployment.Step.deployTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.installTester; import static com.yahoo.vespa.hosted.controller.deployment.Step.report; -import static java.nio.charset.StandardCharsets.UTF_8; import static java.util.Objects.requireNonNull; import static java.util.logging.Level.FINE; import static java.util.logging.Level.INFO; @@ -671,6 +661,7 @@ public class InternalStepRunner implements StepRunner { logger.log(INFO, "The test package should either contain basic HTTP tests under 'tests/<suite-name>/', " + "or a Java test bundle under 'components/' with at least one test with the annotation " + "for this suite. See docs.vespa.ai/en/testing.html for details."); + controller.jobController().updateTestReport(id); return Optional.of(noTests); } case SUCCESS: diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java index 81fbb85e3d7..439f1aa9a09 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArtifactExpirer.java @@ -53,7 +53,7 @@ public class ArtifactExpirer extends ControllerMaintainer { .filter(artifact -> isExpired(artifact, now, versionStatus)) .collect(Collectors.toList()); if (!artifactsToExpire.isEmpty()) { - log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts: " + artifactsToExpire); + log.log(Level.INFO, "Expiring " + artifactsToExpire.size() + " artifacts in " + cloudName + ": " + artifactsToExpire); artifactRegistry.deleteAll(artifactsToExpire); } return 1; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index c94c04fc244..38b9d994a6d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -8,6 +8,7 @@ import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @@ -57,6 +58,8 @@ public class AuditLogSerializer { methodFrom(entryObject.field(methodField)), entryObject.field(resourceField).asString(), SlimeUtils.optionalString(entryObject.field(dataField)) + .map(s -> s.getBytes(StandardCharsets.UTF_8)) + .orElseGet(() -> new byte[0]) )); }); return new AuditLog(entries); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 671222e2123..9bea7fb829d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -1,13 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.proxy; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.text.Text; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; +import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.yolean.concurrent.Sleeper; import org.apache.http.Header; @@ -20,6 +20,7 @@ import org.apache.http.client.methods.HttpPatch; import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpRequestBase; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.InputStreamEntity; import org.apache.http.impl.DefaultConnectionReuseStrategy; import org.apache.http.impl.client.CloseableHttpClient; @@ -29,7 +30,6 @@ import org.apache.http.protocol.HttpCoreContext; import org.apache.http.util.EntityUtils; import javax.net.ssl.HostnameVerifier; -import javax.net.ssl.SSLContext; import javax.net.ssl.SSLSession; import java.io.IOException; import java.io.InputStream; @@ -68,16 +68,15 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C private final Sleeper sleeper; @Inject - public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ServiceIdentityProvider sslContextProvider) { - this(zoneRegistry, sslContextProvider.getIdentitySslContext(), Sleeper.DEFAULT, - new ConnectionReuseStrategy(zoneRegistry)); + public ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, ControllerIdentityProvider identityProvider) { + this(new SSLConnectionSocketFactory(identityProvider.getConfigServerSslSocketFactory(), new ControllerOrConfigserverHostnameVerifier(zoneRegistry)), + Sleeper.DEFAULT, + new ConnectionReuseStrategy(zoneRegistry)); } - ConfigServerRestExecutorImpl(ZoneRegistry zoneRegistry, SSLContext sslContext, + ConfigServerRestExecutorImpl(SSLConnectionSocketFactory connectionSocketFactory, Sleeper sleeper, ConnectionReuseStrategy connectionReuseStrategy) { - this.client = createHttpClient(sslContext, - new ControllerOrConfigserverHostnameVerifier(zoneRegistry), - connectionReuseStrategy); + this.client = createHttpClient(connectionSocketFactory, connectionReuseStrategy); this.sleeper = sleeper; } @@ -227,8 +226,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C } } - private static CloseableHttpClient createHttpClient(SSLContext sslContext, - HostnameVerifier hostnameVerifier, + private static CloseableHttpClient createHttpClient(SSLConnectionSocketFactory connectionSocketFactory, org.apache.http.ConnectionReuseStrategy connectionReuseStrategy) { RequestConfig config = RequestConfig.custom() @@ -237,8 +235,7 @@ public class ConfigServerRestExecutorImpl extends AbstractComponent implements C .setSocketTimeout((int) PROXY_REQUEST_TIMEOUT.toMillis()).build(); return HttpClientBuilder.create() .setUserAgent("config-server-proxy-client") - .setSSLContext(sslContext) - .setSSLHostnameVerifier(hostnameVerifier) + .setSSLSocketFactory(connectionSocketFactory) .setDefaultRequestConfig(config) .setMaxConnPerRoute(10) .setMaxConnTotal(500) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java index c87fea3beb3..4a208aa3794 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/FlagsClient.java @@ -5,10 +5,10 @@ import ai.vespa.util.http.hc4.retry.DelayedConnectionLevelRetryHandler; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.vespa.athenz.api.AthenzIdentity; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.wire.WireErrorResponse; import org.apache.http.HttpEntity; @@ -22,6 +22,7 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPut; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.client.utils.URIBuilder; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.entity.ContentType; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.CloseableHttpClient; @@ -55,7 +56,7 @@ class FlagsClient { private final CloseableHttpClient client; - FlagsClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) { + FlagsClient(ControllerIdentityProvider identityProvider, Set<FlagsTarget> targets) { this.client = createClient(identityProvider, targets); } @@ -95,14 +96,16 @@ class FlagsClient { }); } - private static CloseableHttpClient createClient(ServiceIdentityProvider identityProvider, Set<FlagsTarget> targets) { + private static CloseableHttpClient createClient(ControllerIdentityProvider identityProvider, Set<FlagsTarget> targets) { DelayedConnectionLevelRetryHandler retryHandler = DelayedConnectionLevelRetryHandler.Builder .withExponentialBackoff(Duration.ofSeconds(1), Duration.ofSeconds(20), 5) .build(); + SSLConnectionSocketFactory connectionSocketFactory = new SSLConnectionSocketFactory( + identityProvider.getConfigServerSslSocketFactory(), new FlagTargetsHostnameVerifier(targets)); + return HttpClientBuilder.create() .setUserAgent("controller-flags-v1-client") - .setSSLContext(identityProvider.getIdentitySslContext()) - .setSSLHostnameVerifier(new FlagTargetsHostnameVerifier(targets)) + .setSSLSocketFactory(connectionSocketFactory) .setDefaultRequestConfig(RequestConfig.custom() .setConnectTimeout((int) Duration.ofSeconds(10).toMillis()) .setConnectionRequestTimeout((int) Duration.ofSeconds(10).toMillis()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java index 1b543045adc..abc888abccb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployer.java @@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.controller.restapi.systemflags; import com.yahoo.concurrent.DaemonThreadFactory; import com.yahoo.config.provision.SystemName; import com.yahoo.text.Text; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.flags.FlagId; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.json.FlagData; +import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; import com.yahoo.vespa.hosted.controller.restapi.systemflags.SystemFlagsDeployResult.OperationError; @@ -46,7 +46,7 @@ class SystemFlagsDeployer { private final ExecutorService executor = Executors.newCachedThreadPool(new DaemonThreadFactory("system-flags-deployer-")); - SystemFlagsDeployer(ServiceIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) { + SystemFlagsDeployer(ControllerIdentityProvider identityProvider, SystemName system, Set<FlagsTarget> targets) { this(new FlagsClient(identityProvider, targets), system, targets); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index aaaf09fa781..ed27ffad978 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -8,7 +8,7 @@ import com.yahoo.container.jdisc.ThreadedHttpRequestHandler; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.Path; -import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; +import com.yahoo.vespa.hosted.controller.api.integration.ControllerIdentityProvider; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; @@ -30,7 +30,7 @@ public class SystemFlagsHandler extends ThreadedHttpRequestHandler { @Inject public SystemFlagsHandler(ZoneRegistry zoneRegistry, - ServiceIdentityProvider identityProvider, + ControllerIdentityProvider identityProvider, Executor executor) { super(executor); this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry, true)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 0ecac036913..f4f50de59d7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.Sets; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.Notifications; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; @@ -1177,4 +1176,23 @@ public class ControllerTest { assertEquals(version2, tester.applications().compileVersion(application, OptionalInt.empty())); } + @Test + public void testCloudAccount() { + DeploymentContext context = tester.newDeploymentContext(); + ZoneId zone = ZoneId.from("prod", "us-west-1"); + String cloudAccount = "012345678912"; + var applicationPackage = new ApplicationPackageBuilder() + .cloudAccount(cloudAccount) + .region(zone.region()) + .build(); + try { + context.submit(applicationPackage).deploy(); + fail("Expected exception"); // Account invalid for tenant + } catch (IllegalArgumentException ignored) {} + + tester.controllerTester().flagSource().withListFlag(PermanentFlags.CLOUD_ACCOUNTS.id(), List.of(cloudAccount), String.class); + context.submit(applicationPackage).deploy(); + assertEquals(cloudAccount, tester.controllerTester().configServer().cloudAccount(context.deploymentIdIn(zone)).get().value()); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 27cf1554b4d..5d1a677bf51 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -66,6 +66,7 @@ public class ApplicationPackageBuilder { private boolean explicitSystemTest = false; private boolean explicitStagingTest = false; private Version compileVersion = Version.fromString("6.1"); + private String cloudAccount = null; public ApplicationPackageBuilder majorVersion(int majorVersion) { this.majorVersion = OptionalInt.of(majorVersion); @@ -256,13 +257,23 @@ public class ApplicationPackageBuilder { } } + public ApplicationPackageBuilder cloudAccount(String cloudAccount) { + this.cloudAccount = cloudAccount; + return this; + } + private byte[] deploymentSpec() { StringBuilder xml = new StringBuilder(); xml.append("<deployment version='1.0' "); majorVersion.ifPresent(v -> xml.append("major-version='").append(v).append("' ")); - if(athenzIdentityAttributes != null) { + if (athenzIdentityAttributes != null) { xml.append(athenzIdentityAttributes); } + if (cloudAccount != null) { + xml.append(" cloud-account='"); + xml.append(cloudAccount); + xml.append("'"); + } xml.append(">\n"); for (String instance : instances.split(",")) { xml.append(" <instance id='").append(instance).append("'>\n"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index b3689dacea7..35a12f4b6d4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -4,10 +4,11 @@ package com.yahoo.vespa.hosted.controller.integration; import ai.vespa.http.DomainName; import ai.vespa.http.HttpURL.Path; import ai.vespa.http.HttpURL.Query; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.component.Version; +import com.yahoo.component.annotation.Inject; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; @@ -16,7 +17,6 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.text.Text; import com.yahoo.vespa.flags.json.FlagData; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; @@ -49,7 +49,6 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.net.URI; -import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; @@ -94,6 +93,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map<DeploymentId, Set<ContainerEndpoint>> containerEndpoints = new HashMap<>(); private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); private final Map<DeploymentId, TestReport> testReport = new HashMap<>(); + private final Map<DeploymentId, CloudAccount> cloudAccounts = new HashMap<>(); private List<ProtonMetrics> protonMetrics; private Version lastPrepareVersion = null; @@ -279,6 +279,10 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return Collections.unmodifiableMap(containerEndpoints); } + public Optional<CloudAccount> cloudAccount(DeploymentId deployment) { + return Optional.ofNullable(cloudAccounts.get(deployment)); + } + public Set<String> containerEndpointNames(DeploymentId deployment) { return containerEndpoints.getOrDefault(deployment, Set.of()).stream() .map(ContainerEndpoint::names) @@ -389,6 +393,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer provision(id.zoneId(), id.applicationId(), cluster); this.containerEndpoints.put(id, deployment.containerEndpoints()); + deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account)); if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { putLoadBalancers(id.zoneId(), List.of(new LoadBalancer(UUID.randomUUID().toString(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java index d9e2e61b868..c047f31e171 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializerTest.java @@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import org.junit.Test; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; import java.util.List; -import java.util.Optional; import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; @@ -28,16 +28,19 @@ public class AuditLogSerializerTest { AuditLog log = new AuditLog(List.of( new AuditLog.Entry(i1, "bar", AuditLog.Entry.Method.POST, "/bar/baz/", - Optional.of("0".repeat(2048))), + "0".repeat(2048).getBytes(StandardCharsets.UTF_8)), new AuditLog.Entry(i2, "foo", AuditLog.Entry.Method.POST, "/foo/bar/", - Optional.of("{\"foo\":\"bar\"}")), + "{\"foo\":\"bar\"}".getBytes(StandardCharsets.UTF_8)), new AuditLog.Entry(i3, "baz", AuditLog.Entry.Method.POST, "/foo/baz/", - Optional.of("")), + new byte[0]), new AuditLog.Entry(i4, "baz", AuditLog.Entry.Method.POST, "/foo/baz/", - Optional.of("\ufdff\ufeff\uffff")) // non-ascii + "000\ufdff\ufeff\uffff000".getBytes(StandardCharsets.UTF_8)), // non-ascii + new AuditLog.Entry(i4, "quux", AuditLog.Entry.Method.POST, + "/foo/quux/", + new byte[]{(byte) 0xDE, (byte) 0xAD, (byte) 0xBE, (byte) 0xEF}) // garbage )); AuditLogSerializer serializer = new AuditLogSerializer(); @@ -58,6 +61,7 @@ public class AuditLogSerializerTest { assertEquals(1024, log.entries().get(0).data().get().length()); assertTrue(log.entries().get(2).data().isEmpty()); assertTrue(log.entries().get(3).data().isEmpty()); + assertTrue(log.entries().get(4).data().isEmpty()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java index c4fbf1aa3a5..f5926e799af 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImplTest.java @@ -4,11 +4,10 @@ package com.yahoo.vespa.hosted.controller.proxy; import ai.vespa.http.HttpURL.Path; import com.github.tomakehurst.wiremock.junit.WireMockRule; import com.github.tomakehurst.wiremock.stubbing.Scenario; -import com.yahoo.config.provision.SystemName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.yolean.concurrent.Sleeper; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.protocol.HttpContext; import org.apache.http.protocol.HttpCoreContext; import org.junit.Rule; @@ -39,7 +38,7 @@ public class ConfigServerRestExecutorImplTest { @Test public void proxy_with_retries() throws Exception { var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of("127.0.0.1")); - var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), + var proxy = new ConfigServerRestExecutorImpl(new SSLConnectionSocketFactory(SSLContext.getDefault()), Sleeper.NOOP, connectionReuseStrategy); URI url = url(); @@ -64,8 +63,8 @@ public class ConfigServerRestExecutorImplTest { @Test public void proxy_without_connection_reuse() throws Exception { var connectionReuseStrategy = new CountingConnectionReuseStrategy(Set.of()); - var proxy = new ConfigServerRestExecutorImpl(new ZoneRegistryMock(SystemName.cd), SSLContext.getDefault(), - (duration) -> {}, connectionReuseStrategy); + var proxy = new ConfigServerRestExecutorImpl(new SSLConnectionSocketFactory(SSLContext.getDefault()), + Sleeper.NOOP, connectionReuseStrategy); URI url = url(); String path = url.getPath(); diff --git a/default_build_settings.cmake b/default_build_settings.cmake index c9d8ddea985..0b8cf873a27 100644 --- a/default_build_settings.cmake +++ b/default_build_settings.cmake @@ -44,16 +44,16 @@ function(setup_vespa_default_build_settings_centos_stream_9) set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_rocky_8_5) - message("-- Setting up default build settings for rocky 8.5") +function(setup_vespa_default_build_settings_rocky_8_6) + message("-- Setting up default build settings for rocky 8.6") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) endfunction() -function(setup_vespa_default_build_settings_almalinux_8_5) - message("-- Setting up default build settings for almalinux 8.5") +function(setup_vespa_default_build_settings_almalinux_8_6) + message("-- Setting up default build settings for almalinux 8.6") set(DEFAULT_EXTRA_INCLUDE_DIRECTORY "${VESPA_DEPS}/include" PARENT_SCOPE) - set(DEFAULT_VESPA_LLVM_VERSION "12" PARENT_SCOPE) + set(DEFAULT_VESPA_LLVM_VERSION "13" PARENT_SCOPE) endfunction() function(setup_vespa_default_build_settings_darwin) @@ -206,10 +206,10 @@ function(vespa_use_default_build_settings) setup_vespa_default_build_settings_centos_8() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "centos 9") setup_vespa_default_build_settings_centos_stream_9() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rocky 8.5") - setup_vespa_default_build_settings_rocky_8_5() - elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "almalinux 8.5") - setup_vespa_default_build_settings_almalinux_8_5() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "rocky 8.6") + setup_vespa_default_build_settings_rocky_8_6() + elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "almalinux 8.6") + setup_vespa_default_build_settings_almalinux_8_6() elseif(VESPA_OS_DISTRO STREQUAL "darwin") setup_vespa_default_build_settings_darwin() elseif(VESPA_OS_DISTRO_COMBINED STREQUAL "fedora 34") diff --git a/dist/vespa.spec b/dist/vespa.spec index 029346686a0..a4ede9bac67 100644 --- a/dist/vespa.spec +++ b/dist/vespa.spec @@ -110,7 +110,7 @@ BuildRequires: vespa-gtest = 1.11.0 BuildRequires: vespa-icu-devel >= 65.1.0-1 BuildRequires: vespa-lz4-devel >= 1.9.2-2 BuildRequires: vespa-onnxruntime-devel = 1.11.0 -BuildRequires: vespa-openssl-devel >= 1.1.1n-1 +BuildRequires: vespa-openssl-devel >= 1.1.1o-1 %define _use_vespa_openssl 1 BuildRequires: vespa-protobuf-devel = 3.19.1 BuildRequires: vespa-libzstd-devel >= 1.4.5-2 @@ -126,13 +126,13 @@ BuildRequires: libarchive %if 0%{?_centos_stream} BuildRequires: (llvm-devel >= 13.0.0 and llvm-devel < 14) %else -BuildRequires: (llvm-devel >= 12.0.0 and llvm-devel < 13) +BuildRequires: (llvm-devel >= 13.0.1 and llvm-devel < 14) %endif %else BuildRequires: (llvm-devel >= 12.0.1 and llvm-devel < 13) %endif BuildRequires: vespa-boost-devel >= 1.76.0-1 -BuildRequires: vespa-openssl-devel >= 1.1.1n-1 +BuildRequires: vespa-openssl-devel >= 1.1.1o-1 %define _use_vespa_openssl 1 BuildRequires: vespa-gtest = 1.11.0 %define _use_vespa_gtest 1 @@ -301,7 +301,7 @@ Requires: vespa-gtest = 1.11.0 %if 0%{?_centos_stream} %define _vespa_llvm_version 13 %else -%define _vespa_llvm_version 12 +%define _vespa_llvm_version 13 %endif %else %define _vespa_llvm_version 12 @@ -386,7 +386,7 @@ Requires: vespa-xxhash = 0.8.0 Requires: xxhash-libs >= 0.8.0 %endif %if 0%{?el7} || 0%{?el8} -Requires: vespa-openssl >= 1.1.1n-1 +Requires: vespa-openssl >= 1.1.1o-1 %else Requires: openssl-libs %endif @@ -423,7 +423,7 @@ Requires: vespa-protobuf = 3.19.1 Requires: libicu %endif %if 0%{?el7} || 0%{?el8} -Requires: vespa-openssl >= 1.1.1n-1 +Requires: vespa-openssl >= 1.1.1o-1 %else Requires: openssl-libs %endif @@ -432,7 +432,7 @@ Requires: openssl-libs %if 0%{?_centos_stream} Requires: (llvm-libs >= 13.0.0 and llvm-libs < 14) %else -Requires: (llvm-libs >= 12.0.0 and llvm-libs < 13) +Requires: (llvm-libs >= 13.0.1 and llvm-libs < 14) %endif %else Requires: (llvm-libs >= 12.0.1 and llvm-libs < 13) @@ -620,7 +620,7 @@ rm -rf %{buildroot} %if 0%{?installdir:1} cp -r %{installdir} %{buildroot} %if 0%{?source_base:1} && ! (0%{?amzn2} || 0%{?el7}) -find %{buildroot} -iname '*.so' -print0 | xargs --no-run-if-empty -0 -n1 /usr/lib/rpm/debugedit -b %{source_base} -d %{_builddir}/%{name}-%{version} +find %{buildroot} -exec file {} \; | grep ': ELF ' | cut -d: -f1 | xargs --no-run-if-empty -n1 /usr/lib/rpm/debugedit -b %{source_base} -d %{_builddir}/%{name}-%{version} %endif %else make install DESTDIR=%{buildroot} @@ -822,7 +822,6 @@ fi %dir %{_prefix}/lib64 %{_prefix}/lib64/libfastos.so %{_prefix}/lib64/libfnet.so -%{_prefix}/lib64/libstaging_vespalib.so %{_prefix}/lib64/libvespadefaults.so %{_prefix}/lib64/libvespalib.so %{_prefix}/lib64/libvespalog.so @@ -835,7 +834,6 @@ fi %{_prefix}/lib64 %exclude %{_prefix}/lib64/libfastos.so %exclude %{_prefix}/lib64/libfnet.so -%exclude %{_prefix}/lib64/libstaging_vespalib.so %exclude %{_prefix}/lib64/libvespadefaults.so %exclude %{_prefix}/lib64/libvespalib.so %exclude %{_prefix}/lib64/libvespalog.so diff --git a/docprocs/CMakeLists.txt b/docprocs/CMakeLists.txt index 33f0bdbfa22..30dbb2242d5 100644 --- a/docprocs/CMakeLists.txt +++ b/docprocs/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(docprocs) +install_jar(docprocs-jar-with-dependencies.jar) diff --git a/document/CMakeLists.txt b/document/CMakeLists.txt index 6505de79d39..88dbe2816d9 100644 --- a/document/CMakeLists.txt +++ b/document/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_define_module( fastos vespalog vespalib - staging_vespalib config_cloudconfig vespaeval @@ -43,4 +42,4 @@ vespa_define_module( src/tests/tensor_fieldvalue/partial_remove ) -install_java_artifact(document) +install_jar(document.jar) diff --git a/documentapi/CMakeLists.txt b/documentapi/CMakeLists.txt index 084afb75fe1..9261bcf9114 100644 --- a/documentapi/CMakeLists.txt +++ b/documentapi/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_define_module( document slobrok messagebus - metrics configdefinitions vdslib diff --git a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java index c68560d16b7..189949ceca9 100644 --- a/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java +++ b/documentgen-test/src/test/java/com/yahoo/vespa/config/DocumentGenPluginTest.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.config; -import com.yahoo.compress.CompressionType; import com.yahoo.docproc.DocumentProcessor; import com.yahoo.docproc.Processing; import com.yahoo.docproc.proxy.ProxyDocument; @@ -40,7 +39,7 @@ import com.yahoo.document.serialization.DocumentDeserializerFactory; import com.yahoo.document.serialization.DocumentSerializer; import com.yahoo.document.serialization.DocumentSerializerFactory; import com.yahoo.io.GrowableByteBuffer; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.tensor.Tensor; import com.yahoo.vespa.document.NodeImpl; import com.yahoo.vespa.document.dom.DocumentImpl; diff --git a/eval/CMakeLists.txt b/eval/CMakeLists.txt index ff731158b21..56db263fed4 100644 --- a/eval/CMakeLists.txt +++ b/eval/CMakeLists.txt @@ -2,7 +2,6 @@ vespa_define_module( DEPENDS vespalib - staging_vespalib APPS src/apps/analyze_onnx_model diff --git a/fastlib/.gitignore b/fastlib/.gitignore deleted file mode 100644 index d52d93b8dda..00000000000 --- a/fastlib/.gitignore +++ /dev/null @@ -1,13 +0,0 @@ -.Build_completed -.Dist_completed -.EscrowCopy_completed -.Install_completed -.PostBuild_completed -.PreBuild_completed -.Test_completed -bin -etc -include -lib -update.log -Makefile diff --git a/fastlib/CMakeLists.txt b/fastlib/CMakeLists.txt deleted file mode 100644 index 035c691c04b..00000000000 --- a/fastlib/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - vespalib - - LIBS - src/vespa/fastlib/io - src/vespa/fastlib/io/tests - src/vespa/fastlib/testsuite - src/vespa/fastlib/text - src/vespa/fastlib/text/apps - src/vespa/fastlib/text/tests - src/vespa/fastlib/util - src/vespa/fastlib/util/tests - src/vespa/packages -) diff --git a/fastlib/INSTALL b/fastlib/INSTALL deleted file mode 100644 index 501c4adf550..00000000000 --- a/fastlib/INSTALL +++ /dev/null @@ -1,62 +0,0 @@ -********************************************************************** -** FastLib Build and Installation Guide ** -********************************************************************** - - ----------------------------------------------------------------------- - -How do I build and install the C++ library? - - cd src/cpp - ./configure --fastos-dir <full path to fastos> [--help | <options>] - make -s bootstrap - make -s install - -The default install directory is '../..' (relative to the src/cpp -directory). This means that the library file will be installed in -../../lib/ and the include files in ../../include/fastlib/. -To override the default install directory, use the configure option -'--install-dir'. Try './configure --fastos-dir <fastosdir> --help' -for all available options. - -Support for additional functionality is available when configuring -with --libxml2-dir <full path to libxml2>. -The XML include files must have been installed in -<libxml2-dir>/includes/libxml2/libxml -The XML library must have been installed as -<libxml2-dir>/lib/libxml2.a (or xml2.a on Windows) - - ----------------------------------------------------------------------- - -What kind of make targets are available, and what do they do? - -make depend - generate make-dependancy information -make makefiles - generate makefiles for all modules -make clean - delete all intermediate and output files -make cleandir - does 'make clean' + deletes depend info -make install - install the library and include files - to location $(INSTALLDIR) -make kdoc - generate kdoc documentation -make doxygen - generate doxygen documentation -make bootstrap - (recommended build target) does: - make cleandir - make depend - make makefiles - make -make tests - Compiles test programs, and runs them. - If a test program fails to compile or run, - the make process is stopped. - Please run 'make tests' after changing fastlib code - to ensure you have not broken anything. - ----------------------------------------------------------------------- - - - - - - - - - diff --git a/fastlib/OWNERS b/fastlib/OWNERS deleted file mode 100644 index 885ab949d74..00000000000 --- a/fastlib/OWNERS +++ /dev/null @@ -1,3 +0,0 @@ -havardpe -toregge -baldersheim diff --git a/fastlib/README b/fastlib/README deleted file mode 100644 index 98913493a5b..00000000000 --- a/fastlib/README +++ /dev/null @@ -1 +0,0 @@ -old legacy code that should be removed as soon as we stop using it diff --git a/fastlib/common_config/install.sh b/fastlib/common_config/install.sh deleted file mode 100644 index 0e6395146b0..00000000000 --- a/fastlib/common_config/install.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -cd src/cpp -make FASTOS_DIR=${autobuild_installroot} INSTALL_DIR=${autobuild_installroot} install diff --git a/fastlib/common_config/make.sh b/fastlib/common_config/make.sh deleted file mode 100644 index 66d0cea0b76..00000000000 --- a/fastlib/common_config/make.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -cd src/cpp -make FASTOS_DIR=${autobuild_installroot} INSTALL_DIR=${autobuild_installroot} diff --git a/fastlib/current/release/buildspec b/fastlib/current/release/buildspec deleted file mode 100644 index a546c902603..00000000000 --- a/fastlib/current/release/buildspec +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Config file added by mkspec -# Added by oholsen -$fastlib_about="FAST standard library classes"; -$fastlib_cvs_module="fastlib"; -$fastlib_cvs_tag="current"; -$fastlib_maintainer='mike@fast.no'; -@fastlib_dependencies = ( - "fastos:4.1.1:release", -); -1; diff --git a/fastlib/src/.gitignore b/fastlib/src/.gitignore deleted file mode 100644 index 9d5b23b7747..00000000000 --- a/fastlib/src/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.dsp -*.dsw -*.ncb -*.opt -*.plg -Makefile.ini -config_command.bat -config_command.sh -output diff --git a/fastlib/src/Doxyfile b/fastlib/src/Doxyfile deleted file mode 100644 index da1209eda82..00000000000 --- a/fastlib/src/Doxyfile +++ /dev/null @@ -1,1113 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.3.3 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# General configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = FastLib - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = head - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../../doc/doxygen - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Catalan, Chinese, Chinese-Traditional, Croatian, Czech, Danish, Dutch, -# Finnish, French, German, Greek, Hungarian, Italian, Japanese, Japanese-en -# (Japanese with English messages), Korean, Norwegian, Polish, Portuguese, -# Romanian, Russian, Serbian, Slovak, Slovene, Spanish, Swedish, and Ukrainian. - -OUTPUT_LANGUAGE = English - -# This tag can be used to specify the encoding used in the generated output. -# The encoding is not always determined by the language that is chosen, -# but also whether or not the output is meant for Windows or non-Windows users. -# In case there is a difference, setting the USE_WINDOWS_ENCODING tag to YES -# forces the Windows encoding (this is the default for the Windows binary), -# whereas setting the tag to NO uses a Unix-style encoding (the default for -# all platforms other than Windows). - -USE_WINDOWS_ENCODING = NO - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = YES - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = YES - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = YES - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these classes will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = NO - -# If the HIDE_FRIEND_COMPOUNDS tag is set to YES, Doxygen will hide all -# friend (class|struct|union) declarations. -# If set to NO (the default) these declarations will be included in the -# documentation. - -HIDE_FRIEND_COMPOUNDS = NO - -# If the HIDE_IN_BODY_DOCS tag is set to YES, Doxygen will hide any -# documentation blocks found inside the body of a function. -# If set to NO (the default) these blocks will be appended to the -# function's detailed documentation block. - -HIDE_IN_BODY_DOCS = NO - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = YES - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user-defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = /ld/home/mike/autodoxygen/fastlib/src/cpp - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower-case letters. If set to YES upper-case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are advised to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put a list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = YES - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explict @brief command for a brief description. - -JAVADOC_AUTOBRIEF = NO - -# The MULTILINE_CPP_IS_BRIEF tag can be set to YES to make Doxygen -# treat a multi-line C++ special comment block (i.e. a block of //! or /// -# comments) as a brief description. This used to be the default behaviour. -# The new default is to treat a multi-line C++ comment block as a detailed -# description. Set this tag to YES if you prefer the old behaviour instead. - -MULTILINE_CPP_IS_BRIEF = NO - -# If the DETAILS_AT_TOP tag is set to YES then Doxygen -# will output the detailed description near the top, like JavaDoc. -# If set to NO, the detailed description appears after the member -# documentation. - -DETAILS_AT_TOP = NO - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# reimplements. - -INHERIT_DOCS = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# The GENERATE_DEPRECATEDLIST tag can be used to enable (YES) or -# disable (NO) the deprecated list. This list is created by putting -# \deprecated commands in the documentation. - -GENERATE_DEPRECATEDLIST= YES - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user-defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -ENABLED_SECTIONS = - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consists of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance, some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java sources -# only. Doxygen will then generate output that is more tailored for Java. -# For instance, namespaces will be presented as packages, qualified scopes -# will look different, etc. - -OPTIMIZE_OUTPUT_JAVA = NO - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = YES - -# Set the SUBGROUPING tag to YES (the default) to allow class member groups of -# the same type (for instance a group of public functions) to be put as a -# subgroup of that type (e.g. under the Public Functions section). Set it to -# NO to prevent subgrouping. Alternatively, this can be done per class using -# the \nosubgrouping command. - -SUBGROUPING = YES - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# If WARN_IF_DOC_ERROR is set to YES, doxygen will generate warnings for -# potential errors in the documentation, such as not documenting some -# parameters in a documented function, or documenting parameters that -# don't exist or using markup commands wrongly. - -WARN_IF_DOC_ERROR = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = "$file:$line: $text" - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -INPUT = . - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl *.cs - -FILE_PATTERNS = - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = */tests/* - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse (i.e. when SOURCE_BROWSER is set to YES). - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. - -SOURCE_BROWSER = YES - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = YES - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = Fast_ -IGNORE_PREFIX += FastOS_ -IGNORE_PREFIX += FastS_ - - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = html - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user-defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the CHM_FILE tag can -# be used to specify the file name of the resulting .chm file. You -# can add a path in front of the file if the result should not be -# written to the html output dir. - -CHM_FILE = - -# If the GENERATE_HTMLHELP tag is set to YES, the HHC_LOCATION tag can -# be used to specify the location (absolute path including file name) of -# the HTML help compiler (hhc.exe). If non-empty doxygen will try to run -# the HTML help compiler on the generated index.hhp. - -HHC_LOCATION = - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = NO - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the HTML help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript, DHTML, CSS and frames is required (for instance Mozilla 1.0+, -# Netscape 6.0+, Internet explorer 5.0+, or Konqueror). Windows users are -# probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = NO - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = latex - -# The LATEX_CMD_NAME tag can be used to specify the LaTeX command name to be -# invoked. If left blank `latex' will be used as the default command name. - -LATEX_CMD_NAME = latex - -# The MAKEINDEX_CMD_NAME tag can be used to specify the command name to -# generate index for LaTeX. If left blank `makeindex' will be used as the -# default command name. - -MAKEINDEX_CMD_NAME = makeindex - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = NO - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = NO - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -# If LATEX_HIDE_INDICES is set to YES then doxygen will not -# include the index chapters (such as File Index, Compound Index, etc.) -# in the output. - -LATEX_HIDE_INDICES = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimised for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = rtf - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assigments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = NO - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = man - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = .3 - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_XML = NO - -# The XML_OUTPUT tag is used to specify where the XML pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `xml' will be used as the default path. - -XML_OUTPUT = xml - -# The XML_SCHEMA tag can be used to specify an XML schema, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_SCHEMA = - -# The XML_DTD tag can be used to specify an XML DTD, -# which can be used by a validating XML parser to check the -# syntax of the XML files. - -XML_DTD = - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- - -# If the GENERATE_PERLMOD tag is set to YES Doxygen will -# generate a Perl module file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_PERLMOD = NO - -# If the PERLMOD_LATEX tag is set to YES Doxygen will generate -# the necessary Makefile rules, Perl scripts and LaTeX code to be able -# to generate PDF and DVI output from the Perl module output. - -PERLMOD_LATEX = NO - -# If the PERLMOD_PRETTY tag is set to YES the Perl module output will be -# nicely formatted so it can be parsed by a human reader. This is useful -# if you want to understand what is going on. On the other hand, if this -# tag is set to NO the size of the Perl module output will be much smaller -# and Perl will parse it just the same. - -PERLMOD_PRETTY = YES - -# The names of the make variables in the generated doxyrules.make file -# are prefixed with the string contained in PERLMOD_MAKEVAR_PREFIX. -# This is useful so different doxyrules.make files included by the same -# Makefile don't overwrite each other's variables. - -PERLMOD_MAKEVAR_PREFIX = - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_ONLY_PREDEF tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line, have an all uppercase name, and do not end with a semicolon. Such -# function macros are typically used for boiler-plate code, and will confuse the -# parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES option can be used to specify one or more tagfiles. -# Optionally an initial location of the external documentation -# can be added for each tagfile. The format of a tag file without -# this location is as follows: -# TAGFILES = file1 file2 ... -# Adding location for the tag files is done as follows: -# TAGFILES = file1=loc1 "file2 = loc2" ... -# where "loc1" and "loc2" can be relative or absolute paths or -# URLs. If a location is present for each tag, the installdox tool -# does not have to be run to correct the links. -# Note that each tag file must have a unique name -# (where the name does NOT include the path) -# If a tag file is not located in the directory in which doxygen -# is run, you must also specify the path to the tagfile here. - -TAGFILES = - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = /usr/bin/perl - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in HTML, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superceded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yields more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = YES - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = YES - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If the UML_LOOK tag is set to YES doxygen will generate inheritance and -# collaboration diagrams in a style similiar to the OMG's Unified Modeling -# Language. - -UML_LOOK = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the CALL_GRAPH and HAVE_DOT tags are set to YES then doxygen will -# generate a call dependency graph for every global function or class method. -# Note that enabling this option will significantly increase the time of a run. -# So in most cases it will be better to enable call graphs for selected -# functions only using the \callgraph command. - -CALL_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are png, jpg, or gif -# If left blank png will be used. - -DOT_IMAGE_FORMAT = png - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# The MAX_DOT_GRAPH_DEPTH tag can be used to set the maximum depth of the -# graphs generated by dot. A depth value of 3 means that only nodes reachable -# from the root by following a path via at most 3 edges will be shown. Nodes that -# lay further from the root node will be omitted. Note that setting this option to -# 1 or 2 may greatly reduce the computation time needed for large code bases. Also -# note that a graph may be further truncated if the graph's image dimensions are -# not sufficient to fit the graph (see MAX_DOT_GRAPH_WIDTH and MAX_DOT_GRAPH_HEIGHT). -# If 0 is used for the depth value (the default), the graph is not depth-constrained. - -MAX_DOT_GRAPH_DEPTH = 0 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermediate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO - -# The CGI_NAME tag should be the name of the CGI script that -# starts the search engine (doxysearch) with the correct parameters. -# A script with this name will be generated by doxygen. - -CGI_NAME = search.cgi - -# The CGI_URL tag should be the absolute URL to the directory where the -# cgi binaries are located. See the documentation of your http daemon for -# details. - -CGI_URL = - -# The DOC_URL tag should be the absolute URL to the directory where the -# documentation is located. If left blank the absolute path to the -# documentation, with file:// prepended to it, will be used. - -DOC_URL = - -# The DOC_ABSPATH tag should be the absolute path to the directory where the -# documentation is located. If left blank the directory on the local machine -# will be used. - -DOC_ABSPATH = - -# The BIN_ABSPATH tag must point to the directory where the doxysearch binary -# is installed. - -BIN_ABSPATH = /usr/local/bin/ - -# The EXT_DOC_PATHS tag can be used to specify one or more paths to -# documentation generated for other projects. This allows doxysearch to search -# the documentation for these projects as well. - -EXT_DOC_PATHS = diff --git a/fastlib/src/vespa/fastlib/.gitignore b/fastlib/src/vespa/fastlib/.gitignore deleted file mode 100644 index 0e56cf2f8c1..00000000000 --- a/fastlib/src/vespa/fastlib/.gitignore +++ /dev/null @@ -1 +0,0 @@ -config.h diff --git a/fastlib/src/vespa/fastlib/testsuite/.gitignore b/fastlib/src/vespa/fastlib/testsuite/.gitignore deleted file mode 100644 index 3c74a910ba1..00000000000 --- a/fastlib/src/vespa/fastlib/testsuite/.gitignore +++ /dev/null @@ -1,14 +0,0 @@ -*.So -*.a -*.elc -*.ilk -*.lib -*.o -*.obj -*.pdb -.cvsignore -.depend -.pure -Debug -Makefile -SunWS_cache diff --git a/fastlib/src/vespa/fastlib/testsuite/CMakeLists.txt b/fastlib/src/vespa/fastlib/testsuite/CMakeLists.txt deleted file mode 100644 index 5fe13b21fb9..00000000000 --- a/fastlib/src/vespa/fastlib/testsuite/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_fast_testsuite - SOURCES - suite.cpp - test.cpp - INSTALL lib64 - DEPENDS -) diff --git a/fastlib/src/vespa/fastlib/testsuite/cpptest.el b/fastlib/src/vespa/fastlib/testsuite/cpptest.el deleted file mode 100644 index 31c0906d576..00000000000 --- a/fastlib/src/vespa/fastlib/testsuite/cpptest.el +++ /dev/null @@ -1,2526 +0,0 @@ -;; Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -;; cpptest.el --- C++ unit test support for c++-mode - -;; $Revision: 1.179 $ $Date: 2004-02-17 17:01:15 $ - -;; Author: Nils Sandøy <nils.sandoy@fast.no> - -;; Keywords: C++, tools -;; -;; This file is not part of Emacs -;; -;; Utility functions for unit testing of C++ classes as an addition to the -;; CC mode. - -;; To use this functionality, put the following code in your .emacs file -;; -;; (setq load-path (nconc '( "<TESTSUITE path>" )load-path)) -;; (require 'cpptest) -;; -;; If you desire a different function/frame-key binding (default -;; f4-f8) you should setq the cppt-xxxx-key variables in your .emacs -;; file as well. - -;; This code will look for a 'testproject.el' file in the source code -;; directory, which will be loaded after all variables are -;; defined. This file should be used to set module/project specific -;; variables, like templates and cppt-test-dir. See the sample -;; testproject.el file in this directory. - -;; Notes & issues: -;; -;; The cppt-fund-public-methods function still has some problems with -;; inline methods, most often resulting in the missinterpretation of -;; parts of the inlined method body as method declarations. -;; -;; If you have more than one class declared in the same header file, -;; you might run into problems. Caveat emptor! -;; -;; This code employs some features that are only supported by emacs -;; 21.x and later. Most notably hashes and the push pop methods. -;; -;; I also recommend that you byte-compile this file, to make it run -;; faster. - - -(require 'cppttemplates) - -;; ------------------------------------------------------------------------ -;; User configurable variables -;; ------------------------------------------------------------------------ - -(defvar cppt-use-fastlib-debug-p nil - "*Flag indicating whether to use the fastlib/util/debug.h utility. -This requires fastlib 1.6.2.2+.") - -(defvar cppt-source-location nil - "This variable will be set by cppt-find-builddir to the current source root. -This is to allow cppt-find-srcdir to replace a build path with the original -source path") - -;;(defvar cppt-relative-compile '("~/fbuild/official" -;; "~/fbuild/build/redhat7.3-i686/statusserver-0.0-with-stlport-000") -(defvar cppt-relative-compile nil - "*Substitution to make to convert source directory to build directory. -argument should be nil or a list with two elements, the first element being the -directory prefix to match and the second element the value to replace with. -A value of nil means no conversion") - -(defvar cppt-relative-compile-versions nil - "*Set this to t if the source directory has a versionnumber path element - e.g. -it is on the form <packagename>/nn.nn[.nn]/ (typically 0.0 for development) but -there is no such corresponding element in the build tree (fbuild typical situation). -This path element will then be removed when constructing the build path.") - -(defvar cppt-pretest-target nil - "*Make target that is invoked before each test run") - -(defvar cppt-extra-make-targets nil - "*Extra make targets inserted in the CUSTOMMAKE part of the -fastos.project file in the test directory. This is often used in -conjunction with cppt-pretest-target") - -(defvar cppt-doc-author-p t - "*Should the author name be used in documentation? 'nil' if not.") - -(defvar cppt-use-underscore-p nil - "*If non-nil, use an underscore based naming scheme for methods and -classes. When nil, use uppercase to separate words.") - -(defvar cppt-pretest "$(ECHO_CMD)" - "*Commands to be inserted for PRE_TEST in fastos.project. These -commands will be run before the test executable, and in the same -shell. Typically you'll set your LD_LIBRARY_PATH here. If you use more -than one command, enclose them in a set of curly brackets {} so that -they all execute as a single command.") - -(defvar cppt-posttest "$(ECHO_CMD)" - "*Commands to be inserted for POST_TEST in fastos.project. These -commands will be run before the test executable, and in the same -shell. If you use more than one command, enclose them in a set of -curly brackets {} so that they all execute as a single command.") - -(defvar cppt-test-dir "test" - "*Name of the test directory. This is a subdirectory of the -sourcecode directory.") - -(defvar cppt-LD_LIBRARY_PATH nil - "*This will be prepended to LD_LIBRARY_PATH when executing test binaries") - -(defvar cppt-use-function-keys-flag t - "*Should cppt set function keys? This only affects C++ mode. -If you want to turn this off you must set this variable, before the -(require \'cpptest) statement.") - -(defvar cppt-toggle-header-key [f4] - "*Shortcut key to execute the \\[cppt-toggle-header-src] method. -This only affects C++ mode.") - -(defvar cppt-toggle-interface-key [S-f4] - "*Shortcut key to execute the \\[cppt-toggle-interface-headers] method. -This only affects C++ mode.") - -(defvar cppt-toggle-header-method-key [C-f4] - "*Shortcut key to execute the \\[cppt-toggle-header-src-method] method. -This only affects C++ mode.") - -(defvar cppt-find-other-file-key [C-S-f4] - "*Shortcut key to execute the \\[ff-find-other-file] method. -This only affects C++ mode.") - -(defvar cppt-switch-code-test-key [f5] - "*Shortcut key to execute the \\[cppt-switch-code-test] method. -This only affects C++ mode.") - -(defvar cppt-switch-code-test-method-key [S-C-f5] - "*Shortcut key to execute the \\[cppt-switch-code-test-method] method. -This only affects C++ mode.") - -(defvar cppt-verify-test-methods-key [S-f5] - "*Shortcut key to execute the \\[cppt-verify-test-methods] method. -This only affects C++ mode.") - -(defvar cppt-new-test-method-key [C-f5] - "*Shortcut key to execute the \\[cppt-new-test-method] method. -This only affects C++ mode.") - -(defvar cppt-run-test-key [f6] - "*Shortcut key to execute the \\[cppt-run-test] method. -This only affects C++ mode.") - -(defvar cppt-run-test-debug-key [C-f6] - "*Shortcut key to execute the \\[cppt-run-test-debug] method. -This only affects C++ mode.") - -(defvar cppt-run-single-test-key [S-f2] - "*Shortcut key to execute the \\[cppt-run-single-test] method. -This only affects C++ mode.") - -(defvar cppt-run-single-test-debug-key [C-S-f2] - "*Shortcut key to execute the \\[cppt-run-single-test-debug] method. -This only affects C++ mode.") - -(defvar cppt-run-test-purify-key [S-C-f6] - "*Shortcut key to execute the \\[cppt-run-test-purify] method. -This only affects C++ mode.") - -(defvar cppt-run-test-purify-debug-key [S-M-f6] - "*Shortcut key to execute the \\[cppt-run-test-purify-debug] method. -This only affects C++ mode.") - -(defvar cppt-test-suite-key [f7] - "*Shortcut key to execute the \\[cppt-test-suite] method. -This only affects C++ mode.") - -(defvar cppt-make-test-key [S-f7] - "*Shortcut key to execute the \\[cppt-make-test] method. -This only affects C++ mode.") - -(defvar cppt-suite-debug-key [C-f7] - "*Shortcut key to execute the \\[cppt-suite-debug] method. -This only affects C++ mode.") - -(defvar cppt-suite-purify-key [S-C-f7] - "*Shortcut key to execute the \\[cppt-suite-purify] method. -This only affects C++ mode.") - -(defvar cppt-suite-purify-debug-key [S-M-f7] - "*Shortcut key to execute the \\[cppt-suite-purify-debug] method. -This only affects C++ mode.") - -(defvar cppt-make-plain-key [f8] - "*Shortcut key to execute the \\[cppt-make-plain] method. -This only affects C++ mode.") - -(defvar cppt-make-build-key [S-f8] - "*Shortcut key to execute the \\[cppt-make-build] method. -This only affects C++ mode.") - -(defvar cppt-compile-key [C-f8] - "*Shortcut key to execute the \\[cppt-compile] method. -This only affects C++ mode.") - -(defvar cppt-insert-file-doc-key [S-f9] - "*Shortcut key to execute the \\[cppt-insert-file-doc] method. -This only affects C++ mode.") - -(defvar cppt-insert-class-doc-key [C-f9] - "*Shortcut key to execute the \\[cppt-insert-class-doc] method. -This only affects C++ mode.") - -(defvar cppt-insert-method-doc-key [f9] - "*Shortcut key to execute the \\[cppt-insert-method-doc] method. -This only affects C++ mode.") - -(defvar cppt-insert-copy-disallowed-key [C-f10] - "*Shortcut key to execute the \\[cppt-insert-copy-disallowed] method. -This only affects C++ mode.") - -(defvar cppt-indent-buffer-key [S-iso-lefttab] - "*Shortcut key to execute the \\[cppt-indent-buffer] method. -This only affects C++ mode.") - -(defvar cppt-make-plain-args "\-k " - "* Arguments for make when calling cppt-make-plain.") - -(defvar cppt-use-callback-p nil - "*With newer (3.*) versions of gcc the old callback did not - compile. Introduced a callback in fastlib/util/callback.h and used - this instead. To use it set cppt-use-callback to non nil.") - -;; ------------------------------------------------------------------------ -;; Initialization and include extentions -;; ------------------------------------------------------------------------ - -(defvar cppt-include-source-p "t" - "Should we include the original file being tested in the test -executable? Set this to nil if you are including this as part of a library") - -(defvar cppt-extra-source-includes - "#include \"../Log.h\"" - "String with extra include statements that will be inserted in all -source files. These will also be inserted into the Application and -Suite files. These include statements will be inserted for the -EXTRA_INCLUDES string in the cppt-test-class-app-template and -cppt-suite-template variables") - -(defvar cppt-extra-object-files - '("../Log") - "List of extra object files to be inlcuded in fastos.project for the -test and the suite applications.") - -(defvar cppt-extra-libraries - nil - "List of extra libraries to be inlcuded in fastos.project as -LIBS.") - -(defvar cppt-extra-external-libraries - '("fast") - "List of extra libraries to be inlcuded in fastos.project as -EXTERNALLIBS.") - -(defvar cppt-application-init-code - "FastOS_File::Delete(\"CLASS_NAMETest.log\"); - Fast_FileLogger filelogger(\"CLASS_NAMETest.log\"); - Fast_FileLogger stderrlogger(stderr); - LogDistributor().RegisterDestination(&filelogger, FLOG_ALL); - for (int i=0; i < _argc; ++i) { - if (strcmp(_argv[i], \"-d\") == 0) { - // Turn on debug mode (log to stderr) - LogDistributor().RegisterDestination(&stderrlogger, FLOG_ALL); - LOG_DBG(\"Running in debug mode\"); - } - }" - "Initialization code that is inserted for the INIT_CODE string in the -cppt-test-class-app-template variable") - -(defvar cppt-suite-init-code - "FastOS_File::Delete(\"SUITE.log\"); - Fast_FileLogger filelogger(\"SUITE.log\"); - Fast_FileLogger stderrlogger(stderr); - LogDistributor().RegisterDestination(&filelogger, FLOG_ALL); - for (int i=0; i < _argc; ++i) { - if (strcmp(_argv[i], \"-d\") == 0) { - // Turn on debug mode (log to stderr) - LogDistributor().RegisterDestination(&stderrlogger, FLOG_ALL); - LOG_DBG(\"Running in debug mode\"); - } - }" - "Initialization code that is inserted for the INIT_CODE string in the -cppt-suite-template variable") - -(defvar cppt-test-parameters - "" - "Parameter string to pass the the test executable when running it. -This is inserted where the 'CPPT_TEST_PARAMETERS' string is found in the -cppt-project-template.") - -(defvar cppt-test-dbflags - "-d" - "Parameter string to pass the the test executable when running it -in debug mode. This is inserted where the 'CPPT_TEST_DBFLAGS' string -is found in the cppt-project-template.") - - -;; ---------------------------------------------- -;; Internal utility methods -;; ---------------------------------------------- - -(defun cppt-is-header-file-p (file-name) - "Is file-name a header file ?" - (let* ((idx (string-match "[^.]+$" file-name)) - (ext (substring file-name idx))) - (string-match "h\\(pp\\)?$" ext))) - -(defun cppt-get-parent-dir (dir) - "Retrieve the parent directory of the given directory" -;; (message "Retrieveing parent directory of '%s'" dir) - (let ((parent (if (string-match "\\(.*/\\)[^/]" dir) - (match-string 1 dir) - "/"))) -;; (message "Got parent '%s'" parent) - parent)) - -(defun cppt-buffer-dir-name () - "Return the directory part of the file in the current buffer" - (let* ((cur-file (buffer-file-name)) - (cur-dir (if (string-match "\\(.*/\\)[^/]+$" cur-file) - (match-string 1 cur-file) - "/"))) -;; (message "Current file: '%s'" cur-file) -;; (message "Current directory: '%s'" cur-dir) - cur-dir)) - -(defun cppt-test-method-name (method-name) - "Prefix the named method with test according to the chose naming convention" - (if cppt-use-underscore-p - (concat "test_" method-name) - (concat "test" (cppt-upcase-first-letter method-name)))) - -(defun cppt-replace-token (token replacement) - "Do a buffer wide search replace with a fill-paragraph after each -match" - (let ((old-case-fold case-fold-search)) - (goto-char (point-min)) - (setq case-fold-search nil) - (while (search-forward-regexp token nil t) - (replace-match replacement t t) - (if (string= mode-name "C++") - (c-fill-paragraph))) - (setq case-fold-search old-case-fold))) - -(defun cppt-default-method-name (&optional include-test-prefix-p) - "Make a guess at a default method-name entry. This guess is based -on the text surrounding the cursor." - (let* ((is-header-p (cppt-is-header-file-p buffer-file-name)) - (cw (current-word)) - (word (if (or include-test-prefix-p - (string-match "^test_?\\(.*\\)" cw)) - (match-string 1 cw) cw)) - (regexp-stub "^\\s-*\\(?:[^ (]+\\s-+\\)*%s%s[*&]?\\([^ ()]+\\)\\s-*(\\s-*\\([^(){};]*\\)\\s-*\\(?:throw\\s-*([^)]*)\\)?\\(?:)[^(){};]*[;{]\\|,\\s-*$\\|)\\s-*$\\)") - (regexp-header - (format regexp-stub "\\s-*" - ;; Ignore or include test prefix in method name? - (if include-test-prefix-p "" "\\(?:test_?\\)?"))) - (regexp-src - (format regexp-stub "\\s-*[^ (]+\\s-*::\\s-*" - ;; Ignore or include test prefix in method name? - (if include-test-prefix-p "" "\\(?:test_?\\)?"))) - (regexp (if is-header-p regexp-header regexp-src))) - (save-excursion - (save-restriction - (end-of-line) - (if (not is-header-p) - (c-end-of-statement)) - (end-of-line) - (if (or (search-backward-regexp regexp nil t) - (and (not is-header-p) - ;; Try searching for namespace methods (without ::) - (search-backward-regexp regexp-header nil t))) - (let ((method-name (match-string 1)) - (parameters (match-string 2))) - (message - "Found method declaration for method '%s' with parameters '%s'" - method-name parameters) - (if (string-match "^operator\\([^_].*\\)$" method-name) - (format - "operator%s%s" - (if cppt-use-underscore-p "_" "") - (cppt-get-operator-name - (match-string 1 method-name) parameters)) - method-name)) - word))))) - -(defun cppt-print-default-method-name () - "Print default method name." - (interactive) - (message "Default method name '%s'" (cppt-default-method-name))) - -(defun cppt-upcase-first-letter (str) - "Upcase the first letter of the string argument." - (concat (upcase (substring str 0 1)) (substring str 1))) - -(defun cppt-list-contains-p (lst elem) - "Return t if the list contains element elem." - (or (equal elem (car lst)) - (and lst (cppt-list-contains-p (cdr lst) elem)))) - -(defun cppt-get-src-dir () - "Return the full path of the source directory" - (let* ((reg-ex (format "\\(.*\\)%s/?$" cppt-test-dir)) - (cur-dir (cppt-buffer-dir-name)) - (src-dir (if (string-match reg-ex cur-dir) - (match-string 1 cur-dir) - cur-dir))) - src-dir)) - -(defun cppt-get-test-dir () - "Return the full path of the test directory" - (let* ((reg-ex (format "%s/?$" cppt-test-dir)) - (cur-dir (cppt-buffer-dir-name)) - (test-dir - (if (string-match reg-ex cur-dir) - cur-dir - (format "%s%s/" cur-dir cppt-test-dir)))) - test-dir)) - -(defun cppt-replace-user-name () - "Got point-min and search and replace the USER_NAME string with the -full name of the current user. If the cppt-doc-author-p is nil, the -whole line containing the USER_NAME tag is removed" - (goto-char (point-min)) - (if cppt-doc-author-p - (replace-string "USER_NAME" (user-full-name)) - ;; Remove line with ' * @author USER_NAME' - (while (re-search-forward - "^.*USER_NAME[ \n\r*]*\n\\(\\s-*\\*?\\s-*[^ \n\r]\\)" nil t) - (message "Removing USER_NAME") - (replace-match "\\1" nil nil)))) - -(defun cppt-insert-template (template) - "Insert the given template at current, point and replace common -key words foun in the text. Point at sompletion is at the end of the -inserted text." - (let ((start-point (point)) - (end-point) - (file-name (cppt-strip-path buffer-file-name))) - ;; Insert the template boilerplate - (insert template) - (setq end-point (point)) - (save-restriction - ;; Substitute tags - (narrow-to-region start-point end-point) - (cppt-replace-user-name) - (goto-char (point-min)) - (let ((creation-date nil)) - (while (search-forward "CREATION_DATE" nil t) - (goto-char (match-beginning 0)) - (if (not creation-date) - (setq creation-date (cppt-get-creation-date))) - (message "Replacing CREATION_DATE with '%s'" creation-date) - (replace-string "CREATION_DATE" creation-date))) - (goto-char (point-min)) - (replace-string "YEAR" (format-time-string "%Y")) - (goto-char (point-min)) - (replace-string "FILE_NAME" file-name) - (goto-char (point-min)) - ;; Clear CVS tags - (goto-char (point-min)) - (replace-string "CVS_TAG" "\$\Id\: \$") - (goto-char end-point)))) - -(defun cppt-build-test-method-name (public-method test-methods) - "Build a name for the test method for the named public method. -Append a number to the name if it already exists in the test-method list." - (let ((test-method - (if cppt-use-underscore-p - (concat "test_" public-method) - (concat "test" (cppt-upcase-first-letter public-method)))) - (x)) - (setq x 0) - (while (cppt-list-contains-p test-methods test-method) - (progn - (setq x (+ x 1)) - (setq test-method - (if cppt-use-underscore-p - (format "test_%s%d" public-method x) - (format "test%s%d" - (cppt-upcase-first-letter public-method) x))))) - test-method)) - -(defun cppt-build-suite-name (module-name) - "Build the test suite name for the named module" - (if cppt-use-underscore-p - (concat - (replace-regexp-in-string "_" "" module-name) - "testsuite") - ;; (concat module-name "_test_suite") - (concat (cppt-upcase-first-letter module-name) "TestSuite"))) - -(defun cppt-edit-test (src-file-name dir-name) - "Open the test file for the named file. -Creates a new template for the class if no test file exists." - (let* ((test-file-name - (if (string-match (format "%s/?$" cppt-test-dir) dir-name) - (format "%s/%s%s.cpp" - dir-name src-file-name - (if cppt-use-underscore-p "test" "Test")) - (format "%s%s/%s%s.cpp" - dir-name cppt-test-dir src-file-name - (if cppt-use-underscore-p "test" "Test"))))) -;; (message "Looking for test source file '%s'" test-file-name) - (if (file-exists-p test-file-name) - (find-file test-file-name) - (if (y-or-n-p (format "Add tests for file '%s.h'? " src-file-name)) - (progn - ;; Open the header file for the current buffer before building - ;; a new test file for it - (find-file (cppt-find-header-file-name buffer-file-name)) - (cppt-create-test-file dir-name - src-file-name - test-file-name - (cppt-find-class-name) - (cppt-find-module-name) - (cppt-find-public-method-names)) - ;; Switch back to the new buffer - (find-file test-file-name)))))) - -(defun cppt-create-test-file (dir-name - src-file-name - test-file-name - class-name - module-name - public-methods) - "Create a new test file for the class. -Also add the file to the module test file, alternatively create the module -test file if it does not already exist." - ;; Check whether the target directory exists, create it if not - (let* ((test-dir-name (format "%s/%s" dir-name cppt-test-dir))) - (unless (file-directory-p test-dir-name) - (if (file-exists-p test-dir-name) - (error "Cannot create test directory '%s'. -A file with the same name exists already." test-dir-name) - (progn - (message "Creating test dir '%s'." test-dir-name) - (make-directory test-dir-name) - ;; Create a default .cvsignore file in the new directory - (find-file (format "%s/.cvsignore" test-dir-name)) - (insert cppt-cvsignore-template) - (save-buffer))))) - ;; First build the header file for the new test class - (let ((header-file-name (cppt-find-header-file-name test-file-name))) -;; (message "Creating new test header file %s" header-file-name) - (find-file header-file-name) - ;; (message "Header file created") - (goto-char (point-min)) - ;; (message "Inserting template header body") - (cppt-insert-template cppt-test-header-template) - - (goto-char (point-min)) - (cppt-replace-token "INCLUDE_CALLBACK" - (if cppt-use-callback-p - "#include <fastlib/util/callback.h>" "")) - (goto-char (point-min)) - (message "cpp use callback %s" cppt-use-callback-p) - (cppt-replace-token "CALLBACK_TYPEDEF" - (if cppt-use-callback-p - (concat "typedef fast::util::callback<" - class-name - "_test> tst_method_ptr") - "typedef void(CLASS_NAMETest::* tst_method_ptr) ()")) - - ;; Convert to underscore based naming - (goto-char (point-min)) - ;; (replace-string "CLASS_NAMETest" "CLASS_NAME_test") - (if cppt-use-underscore-p - (cppt-replace-token "CLASS_NAMETest" "CLASS_NAME_test")) - - - (goto-char (point-min)) - (replace-string - "CLASS_NAME.h" - (format "%s.h" (replace-regexp-in-string "_" "" class-name))) - ;; Insert class name - (cppt-replace-token "CLASS_NAME" class-name) - (cppt-insert-test-method-declarations public-methods) - ;; Fill the first paragraph in file header - (goto-char (point-min)) - (search-forward-regexp "[^\n\r /*]") - (c-fill-paragraph) - (save-buffer)) - ;; Build .cpp file for the new application for running the test class - (let ((app-file-name (cppt-find-app-file-name test-file-name))) -;; (message "Creating new test file application %s" app-file-name) - (find-file app-file-name) - (goto-char (point-min)) - (cppt-insert-template cppt-test-class-app-template) - ;; Convert to underscore based naming - (if cppt-use-underscore-p - (progn - (goto-char (point-min)) - (replace-string "CLASS_NAMETestApp" "CLASS_NAME_test_app") - (goto-char (point-min)) - (replace-string "CLASS_NAMETest" "CLASS_NAME_test"))) - (goto-char (point-min)) - (replace-string "TEST_HEADER" - (format "%s%s" - (replace-regexp-in-string "_" "" class-name) - (if cppt-use-underscore-p "test" "Test"))) - ;; DEBUG replace - (goto-char (point-min)) - (cppt-replace-token "DEBUG_INCLUDES" - (if cppt-use-fastlib-debug-p - "#include <fastlib/util/debug.h>" - "")) - (cppt-replace-token "DEBUG_INIT" - (if cppt-use-fastlib-debug-p - "INIT_DEBUG(_argc, _argv);" - "")) - ;; Insert extra include files - (goto-char (point-min)) - (replace-string "EXTRA_INCLUDES" cppt-extra-source-includes) - ;; Insert initialization code - (goto-char (point-min)) - (replace-string "INIT_CODE" - ;; Format argv & argc to _argv and _argc - (replace-regexp-in-string - "\\([^_]\\)argc" "\\1_argc" - (replace-regexp-in-string - "\\([^_]\\)argv" "\\1_argv" - cppt-application-init-code))) - ;; Set the suite name - (cppt-replace-token "MODULE" module-name) - (cppt-replace-token "CLASS_NAME" class-name) - ;; Fill the first paragraph in file header - (goto-char (point-min)) - (search-forward-regexp "[^\n\r /*]") - (c-fill-paragraph) - (save-buffer)) - ;; Build the .cpp file for the new test class -;; (message "Creating new test file %s" test-file-name) - (find-file test-file-name) - (goto-char 1) - (cppt-insert-template cppt-test-body-template) - ;; DEBUG replace - (goto-char (point-min)) - (cppt-replace-token "DEBUG_INCLUDES" - (if cppt-use-fastlib-debug-p - "#include <fastlib/util/debug.h>" - "")) - (if (not cppt-use-fastlib-debug-p) - (cppt-replace-token "DEBUG(.*);" "")) - - ;; callback call replace - (goto-char (point-min)) - (cppt-replace-token "CALL_CALLBACK" - (if cppt-use-callback-p - "itr->second()" - "(this->*itr->second)()")) - ;; Convert to underscore based naming - (goto-char (point-min)) - (and cppt-use-underscore-p - (goto-char (point-min)) - (replace-string "CLASS_NAMETest" "CLASS_NAME_test")) - (goto-char (point-min)) - (replace-string "TEST_HEADER" - (format "%s%s" - (replace-regexp-in-string "_" "" class-name) - (if cppt-use-underscore-p "test" "Test"))) - ;; Insert extra include files - (goto-char (point-min)) - (replace-string "EXTRA_INCLUDES" cppt-extra-source-includes) - (cppt-replace-token "MODULE" module-name) - (cppt-replace-token "CLASS_NAME" class-name) - ;; Fill the first paragraph in file header - (goto-char (point-min)) - (search-forward-regexp "[^\n\r /*]") - (c-fill-paragraph) - (cppt-insert-test-methods public-methods class-name) - (cppt-insert-run-method public-methods class-name) - (save-buffer) - (if (< (count-windows) 2) (split-window-vertically)) - (other-window 1) - (cppt-add-class-test dir-name - test-file-name - src-file-name - class-name - module-name) - (other-window -1)) - -(defun cppt-find-header-file-name (file-name) - "Find the name of the header file for the named file." - (let ((header-file-name - (concat - (substring file-name 0 - (string-match "[^.]+$" file-name)) "h"))) - ;; (message "Header file name: %s" header-file-name) - header-file-name)) - -(defun cppt-find-app-file-name (file-name) - "Build a name for an application file from the given file name" - (let ((app-file-name - (concat - (substring file-name 0 - (string-match "\\.[^.]+$" file-name)) - (if cppt-use-underscore-p "app.cpp" "App.cpp")))) - ;; (message "App file name: %s" app-file-name) - app-file-name)) - -(defun cppt-create-test-suite (file-name suite-name module-name) - "Create a new test suite for the named module." -;; (message "Creating test suite file %s" file-name) - (find-file file-name) - (goto-char (point-min)) - (cppt-insert-template cppt-suite-template) - ;; DEBUG replace - (goto-char (point-min)) - (cppt-replace-token "DEBUG_INCLUDES" - (if cppt-use-fastlib-debug-p - "#include <fastlib/util/debug.h>" - "")) - (cppt-replace-token "DEBUG_INIT" - (if cppt-use-fastlib-debug-p - "INIT_DEBUG(_argc, _argv);" - "")) - ;; Insert extra include files - (goto-char (point-min)) - (replace-string "EXTRA_INCLUDES" cppt-extra-source-includes) - ;; Insert initialization code - (goto-char (point-min)) - (replace-string "INIT_CODE" - ;; Format argv & argc to _argv and _argc - (replace-regexp-in-string - "\\([^_]\\)argc" "\\1_argc" - (replace-regexp-in-string - "\\([^_]\\)argv" "\\1_argv" - cppt-suite-init-code))) - ;; Convert to underscare naming - (and cppt-use-underscore-p - (goto-char (point-min)) - (replace-string "SUITEApp" "SUITE_app")) - (cppt-replace-token "SUITE" suite-name) - (cppt-replace-token "MODULE" module-name) - (save-buffer)) - -(defun cppt-insert-libs (libs) - "Insert one line for each of the items in the libs list" - ;; (message "cppt-insert-libs: %s, num %d" (car libs) (length libs)) - (if libs - (progn - (insert (format "%s\n" (car libs))) - (cppt-insert-libs (cdr libs))) - ;; Insert extra libs, if any - (if cppt-extra-libraries - (let ((lib (car cppt-extra-libraries)) - (rest (cdr cppt-extra-libraries))) - (insert "LIBS ") - (while lib - (progn - (insert lib " ") - (setq lib (car rest)) - (setq rest (cdr rest)))) - (insert "\n"))) - ;; Insert extra external libs, if any - (if cppt-extra-external-libraries - (let ((lib (car cppt-extra-external-libraries)) - (rest (cdr cppt-extra-external-libraries))) - (insert "EXTERNALLIBS ") - (while lib - (progn - (insert lib " ") - (setq lib (car rest)) - (setq rest (cdr rest)))) - (insert "\n"))))) - -(defun cppt-find-project-libs (file-name) - "Build a list with all LIBS or EXTERNALLIBS found in the named project." - ;; (message "Searching for library dependencies within %s" file-name) - (find-file file-name) - (goto-char (point-min)) - (let ((libs)) - (while (search-forward-regexp - "^\\(EXTERNALLIBS\\|LIBS\\).*$" (point-max) t) - (let ((lib-str (match-string 0))) - (unless (cppt-list-contains-p libs lib-str) - (setq libs (cons lib-str libs)) - ;; (message "Found '%s', num libs: %d" (car libs) (length libs)) - ))) - (kill-buffer nil) - libs)) - -(defun cppt-create-project (file-name suite-name) - "Create a new fastos.project file" -;; (message "Creating project file '%s'" file-name) - (find-file file-name) - (goto-char (point-min)) - (cppt-insert-template cppt-project-template) - (if cppt-use-underscore-p - (progn - (goto-char (point-min)) - (replace-string "%Test" "%test") - (goto-char (point-min)) - (replace-string "%testSuite" "%testsuite"))) - (goto-char (point-min)) - (replace-string "EXTRA_MAKE_TARGETS" (or cppt-extra-make-targets "")) - (goto-char (point-min)) - (replace-string "_PRETEST_TARGET_" - (if cppt-pretest-target cppt-pretest-target "")) - (goto-char (point-min)) - (replace-string "_PRE_TEST_" cppt-pretest) - (goto-char (point-min)) - (replace-string "_POST_TEST_" cppt-posttest) - (goto-char (point-min)) - (replace-string "SUITE" suite-name) - (goto-char (point-min)) - (replace-string "CPPT_TEST_PARAMETERS" cppt-test-parameters) - (goto-char (point-min)) - (replace-string "CPPT_TEST_DBFLAGS" cppt-test-dbflags)) - -(defun cppt-insert-test-method-declarations (public-methods &optional comment) - "Insert the declarations of the test methods for all methods -in the given list" - (let ((test-methods) - (test-method)) - (goto-char 1) - (re-search-forward "/[* \n]+Test methods[^/]+/\n" nil t) - (if public-methods - (while public-methods -;; (message "Creating test method declaration for '%s'" -;; (car public-methods)) - (setq test-method - (cppt-build-test-method-name - (car public-methods) test-methods)) - (setq test-methods (cons test-method test-methods)) -;; (message "Inserting declaration of test method '%s'" test-method) - (insert - (format - "\n /**\n * %s\n */\n void %s();\n\n" - (if comment - comment - (format "Test of the '%s' method." (car public-methods))) - test-method)) - (setq public-methods (cdr public-methods))) - ;; No public methods for the class beeing tested - (insert (format " - /** - * This is just a dummy test method to indicate that there are no tests - * for this class - */ - void testTest();\n\n"))))) - -(defun cppt-insert-test-method (test-method class-name public-method - &optional comment) - ;; Verify that the NOTEST macro exists in the file - (save-excursion - (save-restriction - (goto-char (point-min)) - (if (not (search-forward "NOTEST" nil t)) - (progn - (message "Inserting NOTETST macro definition") - (search-forward-regexp "\n/\\*+\n\\s-*\\*\\s-*Test methods" nil t) - (goto-char (match-beginning 0)) - (insert cppt-notest-template))))) - (message "Creating test method '%s' for class '%s'" test-method class-name) - (insert (format " -/** - * %s - */ -void %s%s::%s() { - NOTEST(\"%s\"\); -}\n\n" - (if comment - comment - (format "Test of the '%s' method." public-method)) - class-name - (if (string-match "_?[tT]est$" class-name) - "" - (if cppt-use-underscore-p "_test" "Test")) - test-method - public-method))) - -(defun cppt-insert-test-methods (public-methods class-name &optional comment) - "Insert test methods for all methods in the given list" - (message "Inserting test methods for class %s" class-name) - (let ((test-methods)) - (goto-char 1) - (re-search-forward "/[* \n]+Test methods[^/]+/\n" nil t) - (if public-methods - (while public-methods - (let* ((public-method - (if (listp public-methods) - (car public-methods) - public-methods)) - (test-method - (cppt-build-test-method-name public-method test-methods))) - (setq test-methods (cons test-method test-methods)) - (cppt-insert-test-method test-method - class-name - public-method - comment) - (setq public-methods (cdr public-methods)))) - (progn - (message "No public methods in class '%s'. %s" - "Inserting default test method" class-name) - (insert (format " -/** - * This is just a dummy test method to indicate that there are no tests - * for this class - */ -void %sTest::testTest() { - _fail(\"No tests implemented for class %s\"); -}\n" class-name class-name)))))) - -(defun cppt-insert-run-method (public-methods class-name) - "Insert a run method to execute all test methods in the given list" - (let ((test-methods)) - (goto-char 1) - ;; Try to add to init method first - (if (re-search-forward "::init() {[ \t]*\n" nil t) - (progn - (if public-methods - (while public-methods - (let* ((public-method (if (listp public-methods) - (car public-methods) - public-methods)) - (test-method (cppt-build-test-method-name - public-method test-methods))) - (setq test-methods (cons test-method test-methods)) - (message - "Adding test method '%s' to method container for class '%s'" - test-method class-name) - (insert (format - (concat " test_methods_.\n insert(MethodContainer::value_type\n (std::string(\"%s\"), \n " - (if cppt-use-callback-p - "fast::util::make_callback(*this, &%s%s::%s)));\n" - "&%s%s::%s));\n")) - test-method - class-name - (if (string-match "_?[tT]est$" class-name) - "" - (if cppt-use-underscore-p "_test" "Test")) - test-method)) - (setq public-methods (cdr public-methods)))) - (insert (format - (concat "\n test_methods_[\"test\"] = " - (if cppt-use-callback-p - "fast::util::make_callback(*this, &%s%s::test);\n" - "&%s%s::test;\n")) - class-name - (if (string-match "_?[tT]est$" class-name) - "" - (if cppt-use-underscore-p "_test" "Test")))))) - ;; Use the old way of putting everything in Run instead - (re-search-forward "::Run() {[ \t]*\n" nil t) - (if public-methods - (while public-methods - (let* ((public-method (if (listp public-methods) - (car public-methods) - public-methods)) - (test-method (cppt-build-test-method-name public-method - test-methods))) - (setq test-methods (cons test-method test-methods)) - (message - "Creating run statement for test method %s" test-method) - (insert (format " - if (setUp()) { - %s(); - tearDown(); - }\n" test-method)) - (setq public-methods (cdr public-methods)))) - (insert " - if (setUp()) { - testTest(); - tearDown(); - }\n"))))) - -(defun cppt-insert-extra-objs (extra-objs) - "Add extra OBJS ... descriptions to the current buffer" - (if (car extra-objs) - (progn - (insert (format "OBJS %s\n" (car extra-objs))) - (cppt-insert-extra-objs (cdr extra-objs))))) - -(defun cppt-add-class-test (dir-name - test-file-name - src-file-name - class-name - module-name) - "Add class test to the module test suite and the fastos.project file." - (let* ((src-dir (cppt-get-src-dir)) - (test-name (cppt-strip-file-name test-file-name)) - (src-name (cppt-strip-file-name src-file-name)) - (suite-name (cppt-build-suite-name module-name)) - (file-name (format "%s%s/%s.cpp" - dir-name cppt-test-dir suite-name)) - (project-file-name - (format "%s%s/fastos.project" dir-name cppt-test-dir)) - (test-class-name (concat class-name (if cppt-use-underscore-p - "_test" "Test")))) - ;; Create or open the test suite source file - (if (not (file-exists-p file-name)) - (cppt-create-test-suite file-name suite-name module-name) - (find-file file-name)) - (goto-char (point-min)) - (if (not (search-forward "All tests for this module" nil t)) - (error "Malformed suite template") - (progn -;; (message "Adding %s to suite %s" test-class-name suite-name) - (let ((start (point))) - (insert (format "\n AddTest(new %s());" test-class-name )) - ;; Sort the order of the tests - (sort-lines nil start (search-forward "}"))) - (goto-char (point-min)) - ;; Go to the end of the initial file comment section - (search-forward "*/\n") - (insert (format "#include \"%s.h\"\n" test-name)) - (save-buffer))) - (let ((libs (cppt-find-project-libs - (format "%s/fastos.project" dir-name)))) - (setq suite-name (replace-regexp-in-string "_" "" suite-name)) - ;; Create or open the fastos.project file - (if (not (file-exists-p project-file-name)) - (cppt-create-project project-file-name suite-name) - (find-file project-file-name)) - (goto-char (point-min)) - (if (search-forward (concat "APPLICATION " suite-name) - (point-max) t) - ;; Add class to suite dependencies - (insert (format "\nOBJS %s%s" test-name - (if cppt-include-source-p - (format "\nOBJS ../%s" src-name) - ""))) - (progn - ;; Add suite application, since not present - (goto-char (point-min)) -;; (message "Creating APPLICATION for %s" suite-name) - (insert (format "APPLICATION %s\nOBJS %s\nOBJS %s%s\n" - suite-name suite-name test-name - (if cppt-include-source-p - (format "\nOBJS ../%s" src-name) - ""))) - (cppt-insert-extra-objs cppt-extra-object-files) - (cppt-insert-libs libs))) - ;; Create separate application for the class - (goto-char (point-min)) -;; (message "Creating APPLICATION %s" test-name) - (insert (format "APPLICATION %s\nOBJS %s\nOBJS %s%s\n" - test-name test-name - (format "%s%s" test-name - (if cppt-use-underscore-p "app" "App")) - (if cppt-include-source-p - (format "\nOBJS ../%s" src-name) - ""))) - (cppt-insert-extra-objs cppt-extra-object-files) - (cppt-insert-libs libs) - (insert "\n\n") - (save-buffer) - ;; Generate the makefile by switching back to the source code, - ;; and running make from that directory - (find-file (format "%s/%s.h" src-dir src-file-name)) -;; (message "Generating makefile") - (compile "make makefiles")) - ;; Switch back to the suite file - (find-file file-name))) - -(defun cppt-set-lib-path (lib-path envir) - "Prepend lib-path to the LD_LIBRARY_PATH environent variable." - (let ((new-envir)) - (while (and (car envir) - (not (equal t (compare-strings - (car envir) 0 15 "LD_LIBRARY_PATH" 0 15)))) - (push (pop envir) new-envir)) - (if (car envir) - (progn - ;; Modify existing LD_LIBRARY_PATH entry - (push (format "LD_LIBRARY_PATH=%s:%s" - lib-path (substring (car envir) 16)) - new-envir) - (pop envir) - ;; Add the rest of the environment - (while (car envir) - (push (pop envir) new-envir))) - ;; Add new LD_LIBRARY_PATH entry - (push (format "LD_LIBRARY_PATH=%s" lib-path) new-envir)) - new-envir)) - -(defun cppt-verify-method (method-name - class-name - test-header-file - test-src-file) - "Verify that the test class has a test method for the named method" - (let ((test-method - (if cppt-use-underscore-p - (concat "test_" method-name) - (concat "test" (cppt-upcase-first-letter method-name))))) - ;; Search through the header file to see if the test method exists - (find-file test-header-file) - (goto-char (point-min)) - (if (search-forward (concat test-method "(") nil t) - (message "Test method '%s' exists for method '%s'" - test-method method-name) - (if (not - (y-or-n-p (format "Add test for method '%s'? " method-name))) - (message "Skipping test method '%s'" test-method) - (message "Adding test method '%s'" test-method) - (cppt-insert-test-method-declarations (list method-name)) - (recenter nil) - (find-file test-src-file) - (cppt-insert-test-methods (list method-name) class-name) - (cppt-insert-run-method (list method-name) class-name) - ;; Search to the implementation of the latest method added - (goto-char (point-min)) - (search-forward (format "NOTEST(\"%s\")" method-name)) - (beginning-of-line) - (recenter nil))))) - -(defun cppt-find-class-name () - "Find the first class name of the current buffer. Will use the -name of the file instead if no class can be found" - (interactive) - (save-excursion - (save-restriction - (let ((class-name)) - (goto-char (point-min)) - (if (re-search-forward - "^\\s-*class\\s-+\\([^ \n\r\t;{]+\\)[^;]*{" nil t) - (setq class-name (match-string 1))) - (if class-name - (message "Found class '%s' in '%s'" class-name buffer-file-name) - (progn - (setq class-name (cppt-find-file-name-root)) - (message "No class in '%s', using '%s'" - buffer-file-name class-name))) - class-name)))) - -(defun cppt-get-path (file-name) - "Return just the path of the file-name up to, and including, the last /." - (if (string-match "\\(.*/\\)" file-name) - (match-string 1 file-name) - file-name)) - -(defun cppt-strip-path (file-name) - "Return the last part of the file-name without the leading path." - (let ((regexp ".*/\\(.*\\)")) - (if (string-match regexp file-name) - (match-string 1 file-name) - file-name))) - -(defun cppt-strip-file-name (file-name) - "Strip off leading path and any .h .cpp extension from the given file name" - (let* ((reg-ex ".*/\\(.*\\)") - (stripped-name file-name)) - ;; First strip off any extension - (if (string-match "\\(.*\\)\\..*$" file-name) - (setq stripped-name (match-string 1 file-name))) - (cppt-strip-path stripped-name))) - -(defun cppt-find-file-name-root () - "Find the file name of the current buffer without extension (.h | .cpp)." - (interactive) - (let* ((regexp "[^\\/]+$") - (file-name - (substring buffer-file-name - (string-match regexp buffer-file-name) - (- (string-match "[^.]+$" buffer-file-name) 1)))) - ;; (message "File name: %s" file-name) - file-name)) - -(defun cppt-find-module-name () - "Find the module name of the current buffer." - (interactive) - (save-excursion - (save-restriction - (let* ((dir-name (cppt-buffer-dir-name)) - (re (format "/%s/$" cppt-test-dir)) - (regexp1 "[^\\/]+\\/$") - (regexp2 "\\/$") - (idx (string-match re dir-name)) - (module-name)) - (if idx (setq dir-name (substring dir-name 0 (+ idx 1)))) - (setq module-name (substring dir-name - (string-match regexp1 dir-name) - (string-match regexp2 dir-name))) -;; (message "Module name: %s" module-name) - module-name)))) - -(defun cppt-get-operator-name (operator parameters) - "Find the textual name of the given operator" - (let ((opr-hash (make-hash-table :test 'equal))) - (puthash "+" "plus" opr-hash) - (puthash "-" "minus" opr-hash) - (puthash "*" "star" opr-hash) - (puthash "/" "divide" opr-hash) - (puthash "%" "mod" opr-hash) - (puthash "^" "hat" opr-hash) - (puthash "&" "bitand" opr-hash) - (puthash "|" "bitor" opr-hash) - (puthash "~" "tilde" opr-hash) - (puthash "!" "not" opr-hash) - (puthash "=" "assign" opr-hash) - (puthash "<" "less" opr-hash) - (puthash ">" "greater" opr-hash) - (puthash "++" "increment" opr-hash) - (puthash "--" "decrement" opr-hash) - (puthash "==" "equality" opr-hash) - (puthash "!=" "inequality" opr-hash) - (puthash "<=" "lessequal" opr-hash) - (puthash ">=" "greaterequal" opr-hash) - (puthash "+=" "plusassign" opr-hash) - (puthash "-=" "minusassign" opr-hash) - (puthash "*=" "starassign" opr-hash) - (puthash "/=" "slashassign" opr-hash) - (puthash "%=" "modassign" opr-hash) - (puthash "^=" "hatassign" opr-hash) - (puthash "&=" "andassign" opr-hash) - (puthash "|=" "orassign" opr-hash) - (puthash "<<" "leftshift" opr-hash) - (puthash ">>" "rightsift" opr-hash) - (puthash ">>=" "rightsiftassign" opr-hash) - (puthash "<<=" "leftshiftassign" opr-hash) - (puthash "&&" "and" opr-hash) - (puthash "||" "or" opr-hash) - (puthash "->*" "pointertomember" opr-hash) - (puthash "," "comma" opr-hash) - (puthash "->" "pointer" opr-hash) - (puthash "[]" "squarebracket" opr-hash) - (puthash "()" "parenthesis" opr-hash) - (puthash "new" "new" opr-hash) - (puthash "new[]" "newarray" opr-hash) - (puthash "delete" "delete" opr-hash) - (puthash "delete[]" "deletearray" opr-hash) - (let ((name (gethash operator opr-hash "unknown"))) - (if (or (string= name "increment") (string= name "decrement")) - (if (string= parameters "") - (concat "pre" name) - (concat "post" name)) - name)))) - -(defun cppt-get-operator (operator-name) - "Return a reg-exp for the named operator" - (let ((opr-hash (make-hash-table :test 'equal))) - (puthash "plus" "\\+\\s-*([^}{]+{" opr-hash) - (puthash "minus" "-\\s-*([^}{]+{" opr-hash) - (puthash "star" "\\*\\s-*([^}{]+{" opr-hash) - (puthash "divide" "/\\s-*([^}{]+{" opr-hash) - (puthash "mod" "%\\s-*([^}{]+{" opr-hash) - (puthash "hat" "\\^\\s-*([^}{]+{" opr-hash) - (puthash "bitand" "&\\s-*([^}{]+{" opr-hash) - (puthash "bitor" "|\\s-*([^}{]+{" opr-hash) - (puthash "tilde" "~\\s-*([^}{]+{" opr-hash) - (puthash "not" "!\\s-*([^}{]+{" opr-hash) - (puthash "assign" "=\\s-*([^}{]+{" opr-hash) - (puthash "less" "<\\s-*([^}{]+{" opr-hash) - (puthash "greater" ">\\s-*([^}{]+{" opr-hash) - (puthash "postincrement" "\\+\\+\\s-*(\\s-*int\\s-*)[^}{]*{" opr-hash) - (puthash "preincrement" "\\+\\+\\s-*(\\s-*)[^}{]*{" opr-hash) - (puthash "postdecrement" "--\\s-*(\\s-*int\\s-*)[^}{]*{" opr-hash) - (puthash "predecrement" "--\\s-*(\\s-*)[^}{]*{" opr-hash) - (puthash "equality" "==\\s-*([^}{]+{" opr-hash) - (puthash "inequality" "!=\\s-*([^}{]+{" opr-hash) - (puthash "lessequal" "<=\\s-*([^}{]+{" opr-hash) - (puthash "greaterequal" ">=\\s-*([^}{]+{" opr-hash) - (puthash "plusassign" "\\+=\\s-*([^}{]+{" opr-hash) - (puthash "minusassign" "-=\\s-*([^}{]+{" opr-hash) - (puthash "starassign" "\\*=\\s-*([^}{]+{" opr-hash) - (puthash "slashassign" "/=\\s-*([^}{]+{" opr-hash) - (puthash "modassign" "%=\\s-*([^}{]+{" opr-hash) - (puthash "hatassign" "\\^=\\s-*([^}{]+{" opr-hash) - (puthash "andassign" "&=\\s-*([^}{]+{" opr-hash) - (puthash "orassign" "|=\\s-*([^}{]+{" opr-hash) - (puthash "leftshift" "<<\\s-*([^}{]+{" opr-hash) - (puthash "rightsift" ">>\\s-*([^}{]+{" opr-hash) - (puthash "rightsiftassign" ">>=\\s-*([^}{]+{" opr-hash) - (puthash "leftshiftassign" "<<=\\s-*([^}{]+{" opr-hash) - (puthash "and" "&&\\s-*([^}{]+{" opr-hash) - (puthash "or" "||\\s-*([^}{]+{" opr-hash) - (puthash "pointertomember" "->*\\s-*([^}{]+{" opr-hash) - (puthash "comma" ",\\s-*([^}{]+{" opr-hash) - (puthash "pointer" "->\\s-*([^}{]+{" opr-hash) - (puthash "squarebracket" "\\[\\]\\s-*([^}{]+{" opr-hash) - (puthash "parenthesis" "()\\s-*([^}{]+{" opr-hash) - (puthash "new" "new\\s-*([^}{]+{" opr-hash) - (puthash "newarray" "new\\[\\]\\s-*([^}{]+{" opr-hash) - (puthash "delete" "delete\\s-*([^}{]+{" opr-hash) - (puthash "deletearray" "delete\\[\\]\\s-*([^}{]+{" opr-hash) - (concat "operator\\s-*" (gethash operator-name opr-hash "unknown")))) - -(defun cppt-skip-comments (end) - "Find the first line of non-comments" - ;; Skip // and /* comments - (goto-char - (if (re-search-forward "^\\s-*[^ \t\r\n/][^ \t\r\n/*].*" end t) - (match-beginning 0) - end)) - ;; Skip lines starting with * - (if (< (point) end) - (goto-char - (if (re-search-forward "^\\s-*[^ \t\n\r*].*" end t) - (match-beginning 0) - end)))) - -(defun cppt-find-methods (end-of-region &optional class-name) - "Find all method declarations btw current point and end-of-region" - (let ((methods) - (method) - (end (if end-of-region end-of-region (point-max)))) - (while (< (point) end) - (cppt-skip-comments end) - (if (< (point) end) - ;; Extract method name - (if (not (re-search-forward - "^\\s-+\\(\\([^ /:,{}();\t\n*]\\|[^:]::[^:]\\)+\\)\\(\\([^({},:;]\\|[^:]::[^:]\\)*[ *&\n]+\\)\\([^ *_~][^ :\n;(]*\\)\\s-*(\\s-*\\([^{};]*\\)\\s-*)[^;{/)]*[;{]" end t)) - (goto-char end) - ;; TODO, store the complete match, for documentation - ;; (message "Match #3 %s" (match-string 0)) - (goto-char (match-end 0)) - (let ((leadtok (match-string-no-properties 1)) - (operator-str (match-string-no-properties 3)) - (params (match-string-no-properties 6))) - (setq method (match-string-no-properties 5)) -;; (message "DEBUG: leadtok '%s'" leadtok) -;; (message "DEBUG: operator-str '%s'" operator-str) -;; (message "DEBUG: method '%s'" method) -;; (message "DEBUG: params '%s'" params) - (if (string-match "operator" operator-str) - (setq method (concat "operator" method))) - (if (and class-name (string-equal class-name method)) - (message "Skipping constructor %s" method) - (if (or (string-match "return" leadtok) - (string-match "return" operator-str)) - (message "Skipping return statement %s" method) - (if (string-equal "friend" leadtok) - (message "Skipping friendship declaration %s" - method) - ;; Handle overloaded operators - (if (string-match "operator\\(.*\\)$" method) - (setq method (format "operator_%s" - (cppt-get-operator-name - (match-string 1 method) - params)))) - (setq methods (cons method methods)) - (message "Found %s C++ method %s" - (if class-name "public" "free") method)))))))) - methods)) - -(defun cppt-find-public-method-names () - "Find all public method declarations in the current buffer." - (interactive) - (let ((public-methods) - (method) - (start-of-class) - (end-of-class) - (start-public) - (end-public) - (cur-point) - (class-name) - (indentation) - (type)) - (save-excursion - (save-restriction - (goto-char (point-min)) - (while (< (point) (point-max)) - (setq cur-point (point)) - ;; Find the start of the next class - ;; If class is not found, just move to the end of the file - (if (setq start-of-class - (re-search-forward - "^\\(\\s-*\\)\\(class\\|struct\\)\\s-+\\([^ \n;{]+\\)[^;{]+{" nil 1)) - (progn - (setq indentation (match-string 1)) - (setq type (match-string 2)) - (setq class-name (match-string 3)) - (message "Found class name '%s'" class-name))) - ;; Find all free method declarations before the class declaration - (message "Locating non-class (free) methods") - (goto-char cur-point) - (setq public-methods - (append public-methods (cppt-find-methods start-of-class))) - (if (not start-of-class) - (goto-char (point-max)) - (message "Locating public methods within class '%s'" class-name) - (goto-char start-of-class) - ;; Find }; indented the same way as the class declaration - (setq end-of-class - (or (re-search-forward (format"^%s};" indentation) nil t) - (point-max))) - (goto-char start-of-class) - ;; Loop across all public method declarations in the class - (while (and (setq start-public - (if (string-match "struct" type) - (progn - (setq type "") - start-of-class) - (re-search-forward "^\\s-*public\\s-*:" nil t))) - (< start-public end-of-class)) - (goto-char start-public) - (setq end-public - (or (re-search-forward - "^\\s-*\\(protected\\|private\\)\\s-*:" nil t) - end-of-class)) - (goto-char start-public) - (setq public-methods - (append public-methods - (cppt-find-methods end-public class-name)))) - (goto-char end-of-class))) - public-methods)))) - -(defun cppt-find-builddir (srcdir &optional test-file) - "Return the corresponding build directory for SRCDIR according to -cppt-relative-compile. If cppt-relative-compile is nil, just return SRCDIR" -;; (message "Resolving build directory for directory '%s'" srcdir) - (setq cppt-source-location srcdir) - (let ((build-dir (if cppt-relative-compile - (let* ((src-path (car cppt-relative-compile)) - (build-path (cadr cppt-relative-compile)) - (abbr-src (abbreviate-file-name srcdir))) - (if (string-match src-path abbr-src) - (let ((tmp-build (replace-match build-path t t abbr-src))) - (if (and cppt-relative-compile-versions - (string-match "/[0-9][0-9.]+[0-9]/" - tmp-build)) - (setq tmp-build (replace-match "/" t t tmp-build)) - tmp-build) - tmp-build))) - srcdir))) - (if (not build-dir) (setq build-dir srcdir)) -;; (message "Resolved build dir: '%s'" build-dir) - (if (not test-file) - (setq test-file "Makefile")) - (while (and (file-exists-p build-dir) - (not (file-exists-p (concat build-dir test-file))) - (not (string-equal build-dir "/"))) -;; (message "No '%s' in directory '%s'... trying parent directory" -;; test-file build-dir) - (setq build-dir (cppt-get-parent-dir build-dir))) - build-dir)) - - -(defun cppt-find-containsdir (srcdir &optional test-file) - "Return the first directory above or at this directory that contains -a Makefile or the file test-file if present" - (if (not test-file) - (setq test-file "Makefile")) - (while (and (file-exists-p srcdir) - (not (file-exists-p (concat srcdir test-file))) - (not (string-equal srcdir "/"))) - (message "No '%s' in directory '%s'... trying parent directory" - test-file srcdir) - (setq srcdir (cppt-get-parent-dir srcdir))) - srcdir) - -;; ---------------------------------------------- -;; Interactive test code manipulation methods -;; ---------------------------------------------- - -(defun cppt-new-test-method () - "Interactively insert a new test method" - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let* ((default-name (cppt-default-method-name)) - (default-test-name - (if (stringp default-name) - (concat (if cppt-use-underscore-p "test_" "test") - default-name))) - (input (read-string - (format "Insert new test method%s: " - (if (string= default-test-name "") - "" - (format " (default %s)" default-test-name))))) - (method-name (if (string= input "") - (if (string= default-test-name "") - (error "No method name given") - default-test-name) - input))) - (if (not (string= method-name "")) - ;; Open the test header file - (let* ((default-comment - (format "Test of the '%s' method." method-name)) - (input (read-string - (format "Comment (default \"%s\"): " default-comment))) - (comment (if (string= input "") default-comment input)) - (file-name (cppt-find-file-name-root)) - (dir-name (cppt-buffer-dir-name)) - (class-name)) - (if (string-match "Suite$" file-name) - (error "No corresponding code for test suite %s" file-name) - (progn - (if (string-match "\\(.*\\)\\([Aa]pp\\|[Tt]est\\)$" file-name) - (setq file-name (concat (match-string 1 file-name) - (if cppt-use-underscore-p - "test" "Test"))) - (setq file-name - (concat file-name - (if cppt-use-underscore-p "test" "Test")))) - (if (not (string-match cppt-test-dir dir-name)) - (setq dir-name (concat dir-name cppt-test-dir))) - (setq file-name - (format "%s/%s" dir-name file-name)) - (if (and (file-exists-p (concat file-name ".h")) - (file-exists-p (concat file-name ".cpp"))) - (progn - ;; Insert test declaration in header file - (find-file (concat file-name ".h")) - (setq class-name (cppt-find-class-name)) - (goto-char (point-min)) - (if (re-search-forward - (concat method-name "\\s-*\\s(") nil t) - (error "Method '%s' already exists" method-name) - (progn - (cppt-insert-test-method-declarations - (list method-name) comment) - (recenter nil) - ;; Insert default test implementation in src file - (if (< (count-windows) 2) (split-window-vertically)) - (other-window 1) - (find-file (concat file-name ".cpp")) - (cppt-insert-test-methods - (list method-name) class-name comment) - (cppt-insert-run-method - (list method-name) class-name) - ;; Search to the implementation of the latest method - ;; added - (goto-char (point-min)) - (search-forward - (format "NOTEST(\"%s\")" method-name)) - (beginning-of-line) - (recenter nil)))) - (error "No test code for file '%s'" - buffer-file-name)))))))) - -(defun cppt-verify-test-methods () - "Verify that all public methods in the current class has test methods, -and if not, then interactively ask whether to add tests for each of them." - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let* ((start-file buffer-file-name) - (src-file-name (cppt-find-file-name-root)) - (src-header-file) - (src-dir (cppt-get-src-dir)) - (test-dir (cppt-get-test-dir)) - (test-header-file) - (test-src-file) - (class-name) - (methods) - (idx)) - (if (string-match "Suite$" src-file-name) - (error "No corresponding code for test suite %s" src-file-name) - (if (setq idx (string-match "\\(App\\|Test\\)$" src-file-name)) - (setq src-file-name (substring src-file-name 0 idx))) - ;; Open the header file for the current class - (delete-other-windows) - (setq src-header-file (format "%s/%s.h" src-dir src-file-name)) - (find-file src-header-file) - (setq class-name (cppt-find-class-name)) - (setq methods (cppt-find-public-method-names)) - ;; Open the header and src file for the test code - (setq test-header-file (format "%s/%s%s.h" - test-dir src-file-name - (if cppt-use-underscore-p "test" "Test"))) - (setq test-src-file (format "%s/%s%s.cpp" - test-dir src-file-name - (if cppt-use-underscore-p "test" "Test"))) - (if (not (file-exists-p test-header-file)) - (cppt-switch-code-test) - ;; Now find missing test methods - (let ((missing)) - (while methods - (unless (cppt-verify-method - (car methods) class-name test-header-file test-src-file) - (setq missing t)) - (setq methods (cdr methods))) - (if missing - (find-file test-src-file) - (message "Test methods found for all methods") - (find-file start-file))))))) - -(defun cppt-switch-code-test () - "Switch buffers between class code and class test-code. -If the current buffer holds class code, then the test code is opened. -If the current buffer holds the test code for a class, the code for the class -is opened" - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let* ((idx) - (file-name (cppt-find-file-name-root)) - (directory-name (cppt-buffer-dir-name)) - (src-file) - (hdr-file (concat file-name ".h"))) - (if (string-match "Suite$" file-name) - (error "No corresponding code for test suite %s" file-name) - (progn - (if (setq idx (string-match "[aA]pp$" file-name)) - (setq file-name (substring file-name 0 idx))) - (if (setq idx (string-match "[Tt]est$" file-name)) - (progn - (setq src-file (concat (cppt-get-parent-dir directory-name) - (substring file-name 0 idx) ".cpp")) - (setq hdr-file (concat (cppt-get-parent-dir directory-name) - (substring file-name 0 idx) ".h")) - (find-file (if (file-exists-p src-file) src-file hdr-file))) - (cppt-edit-test file-name directory-name)))))) - -(defun cppt-switch-code-test-method () - "Switch buffers between code- and and test-method. -If the current buffer holds class code, then the test code is opened. -If the current buffer holds the test code for a class, the code for the class -is opened" - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let* ((file-name (cppt-find-file-name-root)) - (is-test-p (or (string-match "[aA]pp$" file-name) - (string-match "[Tt]est$" file-name))) - (default-name (cppt-default-method-name)) - (default-test-name (if (stringp default-name) - (if is-test-p - default-name - (cppt-test-method-name default-name)))) - (method-name (if (string= default-test-name "") - (error "No method name found") - default-test-name))) - (cppt-switch-code-test) - (unless (string= method-name "") - (let* ((is-header-p (string-match "\\.h$" buffer-file-name)) - (reg-exp (if (and is-test-p - (string-match "^operator_?\\(.*\\)" - method-name)) - (cppt-get-operator (match-string 1 method-name)) - (concat method-name "\\s-*([^}{]+{")))) - ;; Go to the named method in the buffer -;; (message "Looking for implementation of method '%s' using reg-exp '%s'" -;; method-name reg-exp) - (goto-char (point-min)) - (if (or (search-forward-regexp (concat "::" reg-exp) nil t) - (and (not is-header-p) - (if is-test-p - ;; Try the header file if no implementation is - ;; found in the source file - (progn - (cppt-toggle-header-src) - (goto-char (point-min)) - (search-forward-regexp reg-exp nil t))))) - (goto-char (match-beginning 0)) - (error "Could not find %smethod '%s'" - (if is-test-p "" "test for ") - method-name)))))) - -(defun cppt-toggle-header-src-method () - "Toggle btw the src and the header file for the current buffer and -go to the current method declaration or implementation. This assumes -that the two files differ only in their extention (.h(pp) .cpp)." - (interactive) - (let* ((is-header-p (cppt-is-header-file-p buffer-file-name)) - (default-name (cppt-default-method-name t)) - (input (read-string - (format "Switch to %s of method%s: " - (if is-header-p "implementation" "declaration") - (if (string= default-name "") - "" - (format " (default %s)" default-name))))) - (method-name (if (string= input "") - (if (string= default-name "") - (error "No method name given") - default-name) - input)) - (regexp (format (if is-header-p "::%s\\s-*(" "\\s-+%s\\s-*(") - method-name))) - (cppt-toggle-header-src) - (goto-char (point-min)) - (if (not (search-forward-regexp regexp nil t)) - (if is-header-p - ;; Try searching for method without :: prefix in case it is - ;; a namespace method - (search-forward-regexp - (format "\\s-+%s\\s-*([^)]*[),]" method-name) nil t))))) - -(defun cppt-toggle-header-src () - "Toggle btw the src and the header file for the current buffer. -This assumes that the two files differ only in their extention (.h(pp) -.cpp)." - (interactive) - ;; There shouldn't be a need for reloading settings here - (let* ((file-name buffer-file-name) - (idx (string-match "[^.]+$" file-name)) - (ext (substring file-name idx)) - (trunk (substring file-name 0 idx)) - (h (concat trunk "h")) - (hpp (concat trunk "hpp")) - (cpp (concat trunk "cpp")) - (target-file-name (if (string-match "cpp" ext) - (if (and (not (file-exists-p h)) - (file-exists-p hpp)) - hpp - h) - cpp))) - (if (or (file-exists-p target-file-name) - (y-or-n-p (format "File '%s' does not exist. Create it? " - target-file-name))) - (find-file target-file-name)))) - -(defun cppt-toggle-interface-headers () - "Toggle btw the header files for the interface and its implementation. -This assumes that the two files differ only in their i prefix" - (interactive) - ;; There should'nt be a need for reloading settings here - (let* ((just-path (cppt-get-path buffer-file-name)) - (just-file-name (cppt-strip-path buffer-file-name)) - (idx (string-match "[^.]+$" just-file-name)) - (prefix (substring just-file-name 0 1)) - (target-file-name - (format "%s%sh" - just-path - (if (string-match "i" prefix) - (substring just-file-name 1 idx) - (concat "i" (substring just-file-name 0 idx)))))) - (if (or (file-exists-p target-file-name) - (y-or-n-p (format "File '%s' does not exist. Create it? " - target-file-name))) - (find-file target-file-name)))) - - -;; ---------------------------------------------- -;; Make and run methods -;; ---------------------------------------------- - -(defun cppt-make-cmd (cmd &optional dir) - "Execute make with cmd as argument" - (let ((old-dir (cppt-buffer-dir-name))) - (if (not dir) (setq dir (cppt-find-builddir old-dir))) - (cd dir) - (message "Executing compile command with args: '%s' within dir: '%s'" - cmd dir) - (if cmd - (compile (concat "make " cmd)) - (call-interactively 'compile)) - (cd old-dir) - (end-of-buffer-other-window nil))) - -(defun cppt-compile () - "Find the lowermost directory with a makefile and interactively run compile" - (interactive) - (let ((compilation-read-command "t")) - (cppt-make-cmd nil))) - -(defun cppt-make (directory-name - module-name - test-file-name - &optional debug - &optional environment) - "Run make within the given directory, then run the given class, -within the the named module." - (let ((dir-name (cppt-find-builddir directory-name)) - (cmd (format "%s %s" - (if environment environment "") - test-file-name)) - (old-process-environment process-environment)) - (if debug (setq cmd (concat cmd debug))) - ;; Temporarily Set LD_LIBRARY_PATH - (if cppt-LD_LIBRARY_PATH - (setq process-environment - (cppt-set-lib-path cppt-LD_LIBRARY_PATH process-environment))) - (cppt-make-cmd cmd dir-name) - (end-of-buffer-other-window nil) - ;; Reset LD_LIBRARY_PATH - (if cppt-LD_LIBRARY_PATH - (setq process-environment old-process-environment)))) - -(defun cppt-make-build () - "Execute make build" - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (cppt-make-cmd "clean all")) - -(defun cppt-make-test () - "Execute make test" - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let* ((old-dir (cppt-buffer-dir-name)) - (directory-name (cppt-find-builddir old-dir))) - (if (or (string-match (format "%s/?$" cppt-test-dir) directory-name) - ;; Also support the old fastlib naming convention - (string-match "tests/$" directory-name)) - (progn - (setq directory-name (cppt-get-parent-dir directory-name)) - (cd directory-name)))) - (cppt-make-cmd "test") - (cd old-dir)) - -(defun cppt-make-plain () - "Execute make without arguments" - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (cppt-make-cmd (if cppt-make-plain-args cppt-make-plain-args ""))) - -(defun cppt-run-test (&optional debug &optional prefix &optional environment) - "Make and execute the test for the current buffer. -The buffer may contain a class to be tested, a class-test or a test suite." - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let ((file-name (cppt-find-file-name-root)) - (module-name (cppt-find-module-name)) - (test-dir (cppt-get-test-dir)) - (idx)) - (unless (or (string-match "[Tt]est$" file-name) - (string-match "[Ss]uite$" file-name)) - (if (setq idx (string-match "[Tt]est[Aa]pp$" file-name)) - (setq file-name (format "%s%s" - (substring file-name 0 idx) - (if cppt-use-underscore-p "test" "Test"))) - (setq file-name - (concat file-name - (if cppt-use-underscore-p "test" "Test"))))) - (cppt-make test-dir module-name - (concat (if cppt-pretest-target - (concat cppt-pretest-target " ")) - (if prefix prefix "run") file-name) - debug - environment))) - -(defun cppt-run-test-debug () - "Execute the test with debug logging" - (interactive) - (cppt-run-test "Debug")) - -(defun cppt-run-single-test (&optional debug &optional prefix) - "Make and execute a single test method for the current buffer. -The buffer may contain a class to be tested, a class-test or a test suite." - (interactive) - (let* ((default-name (cppt-default-method-name)) - (input (read-string - (format "Run test for method%s: " - (if (string= default-name "") - "" - (format " (default %s)" default-name))))) - (method-name (if (string= input "") - (if (string= default-name "") - (error "No method name given") - default-name) - input))) - (if (not (string= method-name "")) - (cppt-run-test - debug prefix - (format "METHOD=%s" - (if cppt-use-underscore-p - (concat "test_" method-name) - (concat "test" (cppt-upcase-first-letter - method-name)))))))) - -(defun cppt-run-single-test-debug () - "Execute a single test method with debug logging" - (interactive) - (cppt-run-single-test "Debug")) - -(defun cppt-run-single-test-purify () - "Execute a single test method with debug logging" - (interactive) - (cppt-run-single-test "" "purify")) - -(defun cppt-run-test-purify () - "Execute the test with purify" - (interactive) - (cppt-run-test "" "purify")) - -(defun cppt-run-test-purify-debug () - "Execute the test with purify in debug mode" - (interactive) - (cppt-run-test "Debug" "purify")) - -(defun cppt-test-suite (&optional debug &optional prefix) - "Make and execute the test suite for the module of the current buffer. -This also works if the current buffer is one of the test suites or classes." - (interactive) - ;; Re-Load any local user configurations - (cppt-load-test-project "testproject.el") - (let ((module-name (cppt-find-module-name)) - (directory-name (cppt-buffer-dir-name))) - ;; Check whether we are in the test directory or the regular - ;; module directory - (unless (string-match (format "/%s/?$" cppt-test-dir) directory-name) - (setq directory-name (concat directory-name "/" cppt-test-dir "/"))) - (cppt-make directory-name module-name - (concat (if cppt-pretest-target - (concat cppt-pretest-target " ")) - (if prefix prefix "run") - (cppt-build-suite-name module-name)) - debug))) - -(defun cppt-suite-debug () - "Make and run the test suite with debug logging" - (interactive) - (cppt-test-suite "Debug")) - -(defun cppt-suite-purify () - "Make and run the test suite with purify" - (interactive) - (cppt-test-suite "" "purify")) - -(defun cppt-suite-purify-debug () - "Make and run the test suite with purify in debug mode" - (interactive) - (cppt-test-suite "Debug" "purify")) - - -;; ---------------------------------------------- -;; Formatting methods -;; ---------------------------------------------- - -(defun cppt-indent (beg end) - (message "Indenting code") - (save-excursion - (save-restriction - (narrow-to-region beg end) - (indent-region (point-min) (point-max) nil) - (untabify (point-min) (point-max)) - (message "Indentation complete")))) - -(defun cppt-indent-buffer () - "Indents and untabifies the current buffer." - (interactive) - (cppt-indent (point-min) (point-max))) - -;; ---------------------------------------------- -;; Automatic code generation methods -;; ---------------------------------------------- - -(defun cppt-insert-copy-disallowed () - "Insert private and unimplemented declarations of the copy CTOR and -assignment operator for the current class" - (interactive) - (end-of-line) - (c-end-of-statement) - ;; Find the class declaration and its name - (if (not - (search-backward-regexp - "\\(?:class\\|struct\\)\\s-+\\(?:<[^>]+>\\s-+\\)?\\(\\w[A-Za-z0-9_]*\\)\\s-*\\(:[^:]\\|{\\|$\\)" - nil t)) - (message "Could not find class declaration") - (let* ((class-name (match-string 1)) - (start-of-class (point)) - (end-of-class (search-forward "};" nil t))) - ;; find first private declaration - (goto-char start-of-class) - (if (not (search-forward-regexp "private\\s-*:" end-of-class t)) - ;; No private declaration, insert one - (progn - (goto-char end-of-class) - (forward-line -1) - (end-of-line) - (insert " -private:"))) - ;; Insert empty and private copy CTOR and assignment operator - (insert (format " - - // Assignment and copy of %s is disallowed, so the following are - // private and unimplemented -%s(const %s&); -%s &operator = (const %s&); -" - class-name class-name class-name class-name class-name)) - (indent-region start-of-class (point) nil) - (search-backward "// Assignment and copy of") - (c-fill-paragraph)))) - -;; ---------------------------------------------- -;; Documentation methods -;; ---------------------------------------------- - -(defun cppt-get-creation-date () - "If the file is registered in CVS use the first registration date, -otherwise use current date" - (save-excursion - (save-restriction - (let ((file buffer-file-name) - (time (current-time))) - (if (vc-registered file) - (progn - (vc-call print-log file) - (set-buffer "*vc*") - (while (not (or (string-match "revision\\s-*1\\.1\\s-*$" - (buffer-string)) - (string-match "added, but not committed" - (buffer-string)))) - (sleep-for 1)) - (if (string-match "added, but not committed" (buffer-string)) - (format-time-string "%d %b %Y" time) - (goto-char (point-min)) - (search-forward-regexp - "revision\\s-*1\\.1[^0-9d]+date:\\s-*\\([0-9]+\\)/0?\\([0-9]+\\)/0?\\([0-9]+\\) 0?\\([0-9]+\\):0?\\([0-9]+\\):0?\\([0-9]+\\)") - (require 'parse-time) - (let* ((year (parse-integer (match-string 1))) - (month (parse-integer (match-string 2))) - (day (parse-integer (match-string 3))) - (hour (parse-integer (match-string 4))) - (min (parse-integer (match-string 5))) - (sec (parse-integer (match-string 6)))) - (setq time (encode-time sec min hour day month year)))))) - (format-time-string "%d %b %Y" time))))) - -(defun cppt-insert-doc-template (template &optional class-name type indent) - "Substitute standard documentation tags found" - (let ((start-point (point)) - (end-point)) - ;; Insert the documentation template - (if indent (insert indent)) - (cppt-insert-template template) - (setq end-point (point)) - (save-restriction - ;; Substitute tags - (narrow-to-region start-point end-point) - (goto-char (point-min)) - (replace-string "TYPE" type) - (goto-char (point-min)) - (replace-string "CLASS_NAME" class-name) - (goto-char (point-min)) - (if indent - (progn - (end-of-line) - (indent-region (point) (point-max) nil) - (goto-char (point-min))))))) - -(defun cppt-insert-file-doc () - "This will insert, a file documentation template at the very top -of the current file." - (interactive) - (let ((class-name nil) (namespace nil) (type)) - (save-excursion - (goto-char (point-min)) - (while (re-search-forward - "^\\s-*namespace\\s-+\\([^ \n\r\t;{]+\\)[^{;]*{" nil t) - (progn - (if namespace - (setq namespace (format "%s::%s" namespace (match-string 1))) - (setq namespace (match-string 1)) -;; (message "Found namespace %s" namespace) - (setq type "namespace") - (goto-char (match-end 0))))) - (goto-char (point-min)) - (if (re-search-forward - "^\\s-*\\(class\\|struct\\)\\s-+\\([^ \n\r\t:;{]+\\)[^;]*{" nil t) - (progn - (setq class-name - (if namespace - (format "%s::%s" namespace (match-string 2)) - (match-string 2))) - (setq type (match-string 1))) -;; (message "Found %s %s" type class-name) - (setq class-name namespace))) - (if (y-or-n-p "Insert file documentation template? ") - (progn - (goto-char (point-min)) - (cppt-insert-doc-template cppt-file-doc-template class-name type) - (re-search-forward "^ \\*\\s-*$") - (if (and (string-match "\\.h$" buffer-file-name) class-name) - (progn - (insert - (format "Header file for the %s %s.\n * " class-name type)) - (search-backward type) - (c-fill-paragraph))))))) - -(defun cppt-insert-class-doc () - "This will insert, a class documentation template in front of the -first class or namespace declaration found from the current point" - (interactive) - (save-excursion - (save-restriction - (beginning-of-line) - (search-forward "};") ;Goto the end of the class declaration - (re-search-backward - "^\\(\\s-*\\)\\(class\\|struct\\|namespace\\)\\s-+\\([^ \n\r\t;{]+\\)[^;]*{" - nil t))) - (let ((class-name (match-string 3)) - (type (match-string 2)) - (indent (match-string 1))) - (if class-name - (progn - (goto-char (match-beginning 0)) - (if (y-or-n-p - (format "Insert documentation template for %s '%s'? " - type class-name)) - (progn - (cppt-insert-doc-template - cppt-class-doc-template class-name type indent) - (re-search-forward class-name) - (end-of-line)))) - (error "No class, struct or namespace found in '%s'" - buffer-file-name)))) - -(defun cppt-build-param-docs (declarations) - "Build a string with documentation of all parameters in the -declarations string" - (if (string-match "\\([^)]+\\))" declarations) - (setq declarations (match-string 1 declarations))) - (let ((documentation "") - (type) (name)) - (while (string-match "\\s-*\\([^=,\n\r]+\\s-+[&*]?\\)\\([^ \t\n\r,=]+\\)[^,)]*,?\\([^{]*\\)" - declarations) - (progn - (setq type (match-string 1 declarations)) - (setq name (match-string 2 declarations)) - (setq declarations (match-string 3 declarations)) - (setq type (replace-regexp-in-string "[\n\r]+" "" type)) - (setq name (replace-regexp-in-string "[\n\r]+" "" name)) - ;; Collapse multipple whitespace - (setq type (replace-regexp-in-string "\\s-\\s-+" " " type)) - ;; Remove trailing whitespace - (setq type (replace-regexp-in-string "\\s-+$" "" type)) - (setq documentation (format "%s * @param %s a '%s' value\n" - documentation name type)))) - documentation)) - -(defun cppt-build-exception-docs (declarations) - "Build a string with documentation of all exceptions in the -declarations string" - (let ((documentation "") - (name)) - (while (and (stringp declarations) - (string-match "\\s-*\\([^=,\n\r]+\\),?\\(.*\\)" - declarations)) - (progn - (setq name (match-string 1 declarations)) - (setq declarations (match-string 2 declarations)) - (setq documentation (format "%s * @exception %s \n" - documentation name)))) - documentation)) - -(defun cppt-insert-method-doc () - "This will insert, a method documentation template in front of the -first method declaration found from the current point" - (interactive) - ;; Search backward to the beginning of the declaration - (end-of-line) - (c-beginning-of-statement) - (beginning-of-line) - (let ((indentation) - (dtor-flag) - (method-name) - (ret-type) - (parameter-declarations) - (start-point (point)) - (method-type) - (exception-declarations)) - (cppt-skip-comments (point-max)) - (if (or - ;; For method declarations - (re-search-forward -;; "^\\(\\s-*\\)\\(\\([^ /:,{}();\t\n*]\\|[^:]::[^:]\\)+\\)\\(\\([^({},:;]\\|[^:]::[^:]\\)*[ *&\n]+\\)\\([^ *_~][^ :\n;(]*\\)\\s-*([^)]*)[^;{/]*[;{]" -;; "^\\(\\s-*\\)\\([^ {};\t\n]*\\s-+[^({};]*\\)\\s-+[*&]?\\s-*\\(~?\\)\\([^ *_~][^ :\n;(]*\\)\\s-*(\\([^;{]*\\))[^);{]*[;{]" - "^\\(\\s-*\\)\\(\\([^/:,{}();\t\n*]\\|[^:]::[^:]\\)+\\)\\s-+[*&]?\\s-*\\(~?\\)\\([^ *_~][^ :\n;(]*\\)\\s-*(\\([^;{)]*\\))[^;{]*[;{]" - (point-max) t) - ;; For method implementations - (and (goto-char start-point) - (re-search-forward - "^\\(\\s-*\\)\\([^({}\n\r\t;]*\\)\\s-+[*&]?[^: \t]+::\\(~?\\)\\([^ *_~][^ :\n;(]*\\)\\s-*(\\([^;{]*\\))[^);{]*[;{]" - (point-max) t))) - (progn - (setq indentation (match-string 1)) - (setq ret-type (match-string 2)) - (setq dtor-flag (match-string 4)) - (setq method-name (match-string 5)) - (setq parameter-declarations (match-string 6)) - ;; Search for exception declarations - (goto-char (match-beginning 0)) - (save-restriction - (save-excursion - (let ((end) - (start (point))) - (search-forward ";") - (setq end (match-beginning 0)) - (goto-char start) - (if (re-search-forward - "throw\\s-*(\\(.*\\))" - end t) - (setq exception-declarations (match-string 1)))))) - ;; Remove method qualifiers from the return type - (setq ret-type (replace-regexp-in-string - "\\(explicit\\|virtual\\|inline\\|static\\)\\s-*" - "" - ret-type)) - ;; Remove trailing whitespace from the return type - (setq ret-type (replace-regexp-in-string "\\s-+$" "" ret-type)) - ;; Remove leading whitespace from the return type - (setq ret-type (replace-regexp-in-string "^\\s-+" "" ret-type)) - (setq method-type - (if (string-equal ret-type "") - (if (string-equal dtor-flag "") - "constructor" - "destructor") - "method")) - ;; Add extra spaces to the indentation that were lost in the regexp - (setq indentation (concat - (if (string-equal ret-type "") " " " ") - indentation)) -;; (message "cppt-insert-method-doc: indentation '%s'" indentation) -;; (message "cppt-insert-method-doc: ret-type '%s'" ret-type) -;; (message "cppt-insert-method-doc: dtor-flag '%s'" dtor-flag) -;; (message "cppt-insert-method-doc: method-name '%s'" method-name) -;; (message "cppt-insert-method-doc: method-type '%s'" method-type) -;; (message "cppt-insert-method-doc: parameter-declarations '%s'" -;; parameter-declarations) -;; (message "cppt-insert-method-doc: exception-declarations '%s'" -;; exception-declarations) - (save-restriction - (save-excursion - ;; Convert method name for overloaded operators - (if (string-match "operator\\(.*\\)" method-name) - (progn - (setq method-name (format "overloaded %s" - (match-string 1 method-name))) - (setq method-type "operator"))))) - (if (y-or-n-p - (format "Insert documentation template for the '%s' %s? " - method-name method-type)) - (let ((end-point)) - (setq start-point (point)) - (save-restriction - ;; Insert the class documentation template - (insert - (format "%s%s" indentation cppt-method-doc-template)) - (setq end-point (point)) - (narrow-to-region start-point end-point) - ;; Substitute the different tags - (goto-char (point-min)) - (replace-string "METHOD_NAME" method-name) - (cppt-replace-user-name) - (goto-char (point-min)) - ;; Trim whitespace - (setq ret-type (replace-regexp-in-string - "[\n\r]+" " " ret-type)) - (setq ret-type (replace-regexp-in-string - "\\s-\\s-+" " " ret-type)) - (setq ret-type (replace-regexp-in-string - "\\s-+$" "" ret-type)) - (replace-string "RETURN_TYPE\n" - (if (or (string-equal ret-type "") - (string-equal ret-type "void")) - "" - (format " * @return a '%s' value\n" - ret-type))) - (goto-char (point-min)) - (replace-string - "PARAMETERS\n" - (cppt-build-param-docs parameter-declarations)) - (goto-char (point-min)) - (replace-string - "EXCEPTIONS\n" - (cppt-build-exception-docs exception-declarations)) - ;; Indent comments properly - (goto-char (point-min)) - (replace-regexp "^\\s-*\\*" (format "%s *" indentation)) - (goto-char (point-max))) - (indent-region (- start-point 1) (point) nil) - (goto-char start-point) - (search-forward method-name) - (end-of-line) - (insert (format "%s " method-type))))) - (error "No method declaration found in '%s'" buffer-file-name)))) - -(defun cppt-cleanup-doc () - "Perform cleanup of documentation strings" - (interactive) - (save-excursion - (save-restriction - (goto-char (point-min)) - (query-replace-regexp "^\\(/\\*\\*\\)\\(\\*+\\|\\s-*\n\\s-*\\*\\*+\\)" - "\\1") - (goto-char (point-min)) - (query-replace-regexp "\\*\\(\\*+\\)\\s-*\n\\s-*\\(\\*/\\)" "\\1\\2") - (goto-char (point-min)) - (query-replace-regexp "\\(@\\(file\\|date\\) \\)\\s-+" "\\1") - (goto-char (point-min)) - (while (search-forward-regexp - "/\\*\\*\\([^/]\\|/[0-9A-Za-z]\\)+\\*/" nil t) - (indent-region (match-beginning 0) (match-end 0) nil))))) - -;; ---------------------------------------------- -;; Load local project definitions -;; ---------------------------------------------- - -(defun cppt-load-test-project (&optional project-file) - "Load the local test project file" - (interactive) - (if (not project-file) - (setq project-file "testproject.el")) - (let* ((directory-name - (cppt-find-containsdir (cppt-buffer-dir-name) project-file)) - (project-file-name)) - (if (string-match (format "%s/$" cppt-test-dir) directory-name) - (setq directory-name (cppt-get-parent-dir directory-name))) - (setq project-file-name (format "%s/%s" directory-name project-file)) - (setq project-file-name - (replace-regexp-in-string "//+" "/" project-file-name)) - ;; Look for the test project file - (if (file-exists-p project-file-name) - (progn -;; (message "Loading local module configuration") - (load-file project-file-name))))) - - -;; --------------------------------------------------------------------- -;; --------------------------------------------------------------------- -;; --------------------------------------------------------------------- -;; Create a minor mode -;; --------------------------------------------------------------------- -;; --------------------------------------------------------------------- -;; --------------------------------------------------------------------- -(defvar c++-test-minor-mode nil - "Mode variable for Fast c++ unit test minor mode.") -(make-variable-buffer-local 'c++-test-minor-mode) - - -(if (not (assq 'c++-test-minor-mode minor-mode-alist)) - (setq minor-mode-alist - (cons '(c++-test-minor-mode " Test") - minor-mode-alist))) - -;; Menu bar -(defvar c++-test-minor-mode-menu-map - (let ((map (make-sparse-keymap "Test"))) - (define-key map [indent-buffer] - '("Indent and untabify buffer" . cppt-indent-buffer)) - - (define-key map [lambda1] '("----")) - - (define-key map [insert-file-doc] - '("Insert doc template for file..." . cppt-insert-file-doc)) - (define-key map [insert-class-doc] - '("Insert doc template for class/struct/namespace..." . - cppt-insert-class-doc)) - (define-key map [insert-method-doc] - '("Insert doc template for method..." . cppt-insert-method-doc)) - - (define-key map [lambda2] '("----")) - - (define-key map [insert-copy-disallowed] - '("Insert empty copy CTOR and assignment operator..." . - cppt-insert-copy-disallowed)) - - (define-key map [lambda3] '("----")) - - (define-key map [cppt-compile] - '("Run make with args..." . cppt-compile)) - (define-key map [make-build] - '("Run 'make clean all'" . cppt-make-build)) - (define-key map [make-plain] - '("Run 'make without arguments'" . cppt-make-plain)) - (define-key map [make-test] - '("Run 'make test'" . cppt-make-test)) - - (define-key map [lambda4] '("----")) - - (define-key map [suite-purify-debug] - '("Run test suite in Purify in debug mode" . cppt-suite-purify-debug)) - (define-key map [suite-purify] - '("Run test suite in Purify" . cppt-suite-purify)) - (define-key map [suite-debug] - '("Run test suite in debug mode" . cppt-suite-debug)) - (define-key map [test-suite] - '("Run test suite" . cppt-test-suite)) - - (define-key map [lambda5] '("----")) - - (define-key map [run-test-purify-debug] - '("Run tests for class in Purify in debug mode" . - cppt-run-test-purify-debug)) - (define-key map [run-test-purify] - '("Run tests for class in Purify" . cppt-run-test-purify)) - (define-key map [run-test-debug] - '("Run tests for class in debug mode" . cppt-run-test-debug)) - (define-key map [run-test] - '("Run tests for class" . cppt-run-test)) - - (define-key map [lambda6] '("----")) - - (define-key map [run-single-test-debug] - '("Run a single test in debug mode..." . cppt-run-single-test-debug)) - (define-key map [run-single-test] - '("Run a single test for class..." . cppt-run-single-test)) - - (define-key map [lambda7] '("----")) - - (define-key map [new-test-method] - '("Create a new test method..." . cppt-new-test-method)) - (define-key map [verify-test-methods] - '("Verify tests for all public methods" . cppt-verify-test-methods)) - - (define-key map [lambda8] '("----")) - - (define-key map [switch-code-test] - '("Toggle test- and source-file" . cppt-switch-code-test)) - (define-key map [switch-code-test-method] - '("Toggle test- and source-method" . cppt-switch-code-test-method)) - (define-key map [toggle-header-src] - '("Toggle header and source" . cppt-toggle-header-src)) - (define-key map [toggle-header-src-method] - '("Toggle header and source with regards to the current method" . - cppt-toggle-header-src-method)) - (define-key map [ff-find-other-file] - '("Find related file, taking includes into consideration" . - ff-find-other-file)) - (define-key map [toggle-interface-headers] - '("Toggle header files for interface and implementation" . - cppt-toggle-interface-headers)) - map) - "Menu for C++ test minor mode") - -;; ---------------------------------------------------------------------- -;; Key bindings, the user should set the cppt-xxx-key variables -;; ---------------------------------------------------------------------- -(defvar c++-test-minor-keymap - (let ((map (make-sparse-keymap))) - (define-key map [menu-bar test] - (cons "Test" c++-test-minor-mode-menu-map)) - (if cppt-use-function-keys-flag - (progn - (define-key map cppt-indent-buffer-key 'cppt-indent-buffer) - (define-key map cppt-insert-file-doc-key 'cppt-insert-file-doc) - (define-key map cppt-insert-class-doc-key 'cppt-insert-class-doc) - (define-key map cppt-insert-method-doc-key 'cppt-insert-method-doc) - (define-key map cppt-insert-copy-disallowed-key - 'cppt-insert-copy-disallowed) - (define-key map cppt-compile-key 'cppt-compile) - (define-key map cppt-make-build-key 'cppt-make-build) - (define-key map cppt-make-plain-key 'cppt-make-plain) - (define-key map cppt-make-test-key 'cppt-make-test) - (define-key map cppt-suite-purify-debug-key 'cppt-suite-purify-debug) - (define-key map cppt-suite-purify-key 'cppt-suite-purify) - (define-key map cppt-suite-debug-key 'cppt-suite-debug) - (define-key map cppt-test-suite-key 'cppt-test-suite) - (define-key map cppt-run-test-purify-debug-key - 'cppt-run-test-purify-debug) - (define-key map cppt-run-test-purify-key 'cppt-run-test-purify) - (define-key map cppt-run-test-debug-key 'cppt-run-test-debug) - (define-key map cppt-run-test-key 'cppt-run-test) - (define-key map cppt-run-single-test-debug-key - 'cppt-run-single-test-debug) - (define-key map cppt-run-single-test-key 'cppt-run-single-test) - (define-key map cppt-new-test-method-key 'cppt-new-test-method) - (define-key map cppt-verify-test-methods-key - 'cppt-verify-test-methods) - (define-key map cppt-switch-code-test-key 'cppt-switch-code-test) - (define-key map cppt-switch-code-test-method-key - 'cppt-switch-code-test-method) - (define-key map cppt-toggle-header-method-key - 'cppt-toggle-header-src-method) - (define-key map cppt-find-other-file-key 'ff-find-other-file) - (define-key map cppt-toggle-interface-key - 'cppt-toggle-interface-headers) - (define-key map cppt-toggle-header-key 'cppt-toggle-header-src) - (define-key map [S-right] 'forward-sexp) - (define-key map [S-left] 'backward-sexp) - (define-key map [S-up] 'beginning-of-defun) - (define-key map [S-down] 'end-of-defun))) - map) - "Keymap used for the c++ test minor mode") - -(or (not (boundp 'minor-mode-map-alist)) - (assoc 'c++-test-minor-mode minor-mode-map-alist) - (setq minor-mode-map-alist - (cons (cons 'c++-test-minor-mode c++-test-minor-keymap) - minor-mode-map-alist))) - -(defun c++-test-minor-mode (&optional arg) - "C++ unit test minor mode. This minor mode is invoked automatically -as an extension of c++-mode. It has extensive functionality for -writing, extending and running automated unit tests for C++ code. - -The functionality may be roughly separated into three different areas: - * Writing and extending unit tests - * Executing the tests - * Documenting code - -The test commands all have the cppt- prefix, and are by default tied -to the different function keys: - -\\{c++-test-minor-keymap} -" - (interactive "P") - (setq c++-test-minor-mode - (if (null arg) - (not c++-test-minor-mode) - (> (prefix-numeric-value arg) 0))) - (if c++-test-minor-mode - (progn - (message "Enabling c++ unit test minor mode") - ;; Load any local user configurations - ;; This is done every time c++-mode is invoked on a file. - (cppt-load-test-project "testproject.el")))) - - -;; Always use this minor mode for c++-mode -(add-hook 'c++-mode-hook 'c++-test-minor-mode) - -(defun cppt-makefile-make-interactive () - "Execute compile with argument taken from current word" - (interactive) - (let* ((regexp "^\\(\\w+\\)\\s-*:") - (compile-command (concat "make " - (progn - (end-of-line) - (if (or (search-backward-regexp regexp nil t) - (search-forward-regexp regexp nil t)) - (match-string 1) - "")))) - (compilation-read-command "t")) - (call-interactively 'compile) - (end-of-buffer-other-window nil))) - -(defun cppt-makefile-mode-hook () - (local-set-key cppt-compile-key 'cppt-makefile-make-interactive) - (local-set-key cppt-make-build-key 'cppt-make-build) - (local-set-key cppt-make-plain-key 'cppt-make-plain) - (local-set-key cppt-make-test-key 'cppt-make-test) - (local-set-key cppt-suite-purify-debug-key 'cppt-suite-purify-debug) - (local-set-key cppt-suite-purify-key 'cppt-suite-purify) - (local-set-key cppt-suite-debug-key 'cppt-suite-debug) - (local-set-key cppt-test-suite-key 'cppt-test-suite)) -(add-hook 'makefile-mode-hook 'cppt-makefile-mode-hook) - - -(provide 'cpptest) diff --git a/fastlib/src/vespa/fastlib/testsuite/suite.cpp b/fastlib/src/vespa/fastlib/testsuite/suite.cpp deleted file mode 100644 index e69de29bb2d..00000000000 --- a/fastlib/src/vespa/fastlib/testsuite/suite.cpp +++ /dev/null diff --git a/fastlib/src/vespa/fastlib/testsuite/testproject.el b/fastlib/src/vespa/fastlib/testsuite/testproject.el deleted file mode 100644 index 983bd582a8a..00000000000 --- a/fastlib/src/vespa/fastlib/testsuite/testproject.el +++ /dev/null @@ -1,108 +0,0 @@ -;; Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -;; testproject.el - -;; Local configurations for the cpptest Emacs unit-test -;; framework. This is just an example of typical variables that one -;; usually uses This file should be located in the same directory as -;; the class(es) you want to test. - -;; $Revision: 1.6 $ $Date: 2003-09-11 09:14:01 $ -;; Author: Nils Sandøy <nils.sandoy@fast.no> - -;; Just a message to show that this file is beeing read. Look for this -;; in the *Messages* buffer. -(message "Setting local test configuration for the module") - -;; Use fastlib's new debugging features. -;; This requires fastlib 1.6.2.2+ -(setq cppt-use-fastlib-debug-p t) - -;; Use an underscore based naming scheme classes and method names will -;; Upcase each word instead -(setq cppt-use-underscore-p t) - -;; Use author in documentation. Set this value to nil if not -(setq cppt-doc-author-p t) - -;; Use the new fastlib callback method. -;; Use this for newer versions of gcc -(setq cppt-use-callback-p t) - -;; This is a subdirectory of the directory in which this file, along with -;; the source code to test, resides -(setq cppt-test-dir "test") - -;; Use this variable to include extra file in your test source, and -;; application files. Typically this will hold headers for log -;; functionality etc. -;; Example: (setq cppt-extra-source-includes "#include \"../Log.h\"") -(setq cppt-extra-source-includes "") - -;; If the above source files are not part of a library, you will -;; probably have to include them in the fastos.project file. -;; Example: (setq cppt-extra-object-files '("../Log")) -(setq cppt-extra-object-files nil) - -;; If the source code does not have a fastos.project file with all -;; required libraries for linking an executable (typically the case -;; when the source is part of a library itself), then you should use -;; this variable to provide a list of libraries which will be appended -;; to the EXTERNALLIBS section for all applications in the -;; fastos.project file. -;; Example: (setq cppt-extra-libraries '("fast")) -(setq cppt-extra-libraries nil) -(setq cppt-extra-external-libraries '("fast")) - -;; Include source file in test executables. -;; Set this to nil if you are testing part of a library -(setq cppt-include-source-p "t") - - -;; If your initialisation code below requires special parameters for -;; running the test executables, add them here -;; Example: (setq cppt-test-parameters "--test-mode") -(setq cppt-test-parameters "") - -;; If you support a special debug mode, which is executed through the -;; cppt-suite-debug or cppt-run-test-debug methods, then you should -;; add the parameter for identifying this here. -;; The parameters given here assume that the fastlib debug features are -;; turned on -;; Example: (setq cppt-test-dbflags "-d") -(setq cppt-test-dbflags "-d all -d emacs") - -;; If you support logging etc, you should include code here for -;; insitializing this as part of the Main body of the test application -;; Example: -;; Add intialization code that turns on logging, and logs to stderr in debug -;; mode -;; (setq cppt-application-init-code -;; "RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(\"CLASSTest.log\"), FLOG_ALL); -;; for (int i=0; i < argc; ++i) { -;; if (strcmp(argv[i], \"-d\") == 0) { -;; // Turn on debug mode (log to stderr) -;; RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(stderr), FLOG_ALL); -;; LOG_DBG(\"Running in debug mode\"); -;; } -;; }") -(setq cppt-application-init-code "") - -;; Pretty much the same as the application init code, but this is used -;; for the Main method of the test suite. -;; Example: -;; Add intialization code that turns on logging, and logs to stderr in debug -;; mode -;; (setq cppt-suite-init-code -;; "RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(\"SUITETest.log\"), FLOG_ALL); -;; for (int i=0; i < argc; ++i) { -;; if (strcmp(argv[i], \"-d\") == 0) { -;; // Turn on debug mode (log to stderr) -;; RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(stderr), FLOG_ALL); -;; LOG_DBG(\"Running in debug mode\"); -;; } -;; }") -(setq cppt-suite-init-code "") diff --git a/fastlib/src/vespa/fastlib/text/normwordfolder.cpp b/fastlib/src/vespa/fastlib/text/normwordfolder.cpp deleted file mode 100644 index a063332e3d1..00000000000 --- a/fastlib/src/vespa/fastlib/text/normwordfolder.cpp +++ /dev/null @@ -1,1195 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/fastlib/text/unicodeutil.h> -#include <vespa/fastlib/text/normwordfolder.h> -#include <mutex> -#include <cstring> - -bool Fast_NormalizeWordFolder::_isInitialized = false; -std::mutex _initMutex; -bool Fast_NormalizeWordFolder::_doAccentRemoval = false; -bool Fast_NormalizeWordFolder::_doSmallToNormalKana = false; -bool Fast_NormalizeWordFolder::_doKatakanaToHiragana = false; -bool Fast_NormalizeWordFolder::_doKanaAccentCollapsing = false; -bool Fast_NormalizeWordFolder::_doFullwidthToBasicLatin = false; -bool Fast_NormalizeWordFolder::_doSharpSSubstitution = false; -bool Fast_NormalizeWordFolder::_doLigatureSubstitution = false; -bool Fast_NormalizeWordFolder::_doMulticharExpansion = false; -bool Fast_NormalizeWordFolder::_isWord[128]; - -ucs4_t Fast_NormalizeWordFolder::_foldCase[767]; // Up to Latin Extended B (0x0250) -ucs4_t Fast_NormalizeWordFolder::_keepCase[767]; // -ucs4_t Fast_NormalizeWordFolder::_foldCaseHighAscii[256]; // Latin Extended Additional (0x1E00 - 0x1F00) -ucs4_t Fast_NormalizeWordFolder::_keepCaseHighAscii[256]; // (incl. vietnamese) -ucs4_t Fast_NormalizeWordFolder::_kanaMap[192]; -ucs4_t Fast_NormalizeWordFolder::_halfwidth_fullwidthMap[240]; - -void -Fast_NormalizeWordFolder::Setup(uint32_t flags) -{ - // Only allow setting these when not initialized or initializing... - { - std::lock_guard<std::mutex> initGuard(_initMutex); - _doAccentRemoval = (DO_ACCENT_REMOVAL & flags) != 0; -// _doSmallToNormalKana = (DO_SMALL_TO_NORMAL_KANA & flags) != 0; -// _doKatakanaToHiragana = (DO_KATAKANA_TO_HIRAGANA & flags) != 0; -// _doKanaAccentCollapsing = (DO_KANA_ACCENT_COLLAPSING & flags) != 0; // Not implemented - _doFullwidthToBasicLatin = (DO_FULLWIDTH_TO_BASIC_LATIN & flags) != 0; // Not implemented - _doSharpSSubstitution = (DO_SHARP_S_SUBSTITUTION & flags) != 0; - _doLigatureSubstitution = (DO_LIGATURE_SUBSTITUTION & flags) != 0; - _doMulticharExpansion = (DO_MULTICHAR_EXPANSION & flags) != 0; - _isInitialized = false; - } - Initialize(); -} - -void -Fast_NormalizeWordFolder::Initialize() -{ - unsigned int i; - if (!_isInitialized) { - std::lock_guard<std::mutex> initGuard(_initMutex); - if (!_isInitialized) { - - for (i = 0; i < 128; i++) - _isWord[i] = Fast_UnicodeUtil::IsWordChar(i); - for (i = 0; i < 767; i++) { - _foldCase[i] = Fast_UnicodeUtil::ToLower(i); - _keepCase[i] = i; - } - - for (i = 0x1E00; i < 0x1F00; i++) { - _foldCaseHighAscii[i - 0x1E00] = Fast_UnicodeUtil::ToLower(i); - _keepCaseHighAscii[i - 0x1E00] = i; - } - - if (_doAccentRemoval) { - _foldCase[0xc0] = 'a'; - _foldCase[0xc1] = 'a'; - _foldCase[0xc2] = 'a'; - _foldCase[0xc3] = 'a'; // A tilde - _foldCase[0xc7] = 'c'; - _foldCase[0xc8] = 'e'; - _foldCase[0xc9] = 'e'; - _foldCase[0xca] = 'e'; - _foldCase[0xcb] = 'e'; - _foldCase[0xcc] = 'i'; // I grave - _foldCase[0xcd] = 'i'; - _foldCase[0xce] = 'i'; - _foldCase[0xcf] = 'i'; - _foldCase[0xd1] = 'n'; - _foldCase[0xd2] = 'o'; - _foldCase[0xd3] = 'o'; - _foldCase[0xd4] = 'o'; - _foldCase[0xd5] = 'o'; - _foldCase[0xd9] = 'u'; - _foldCase[0xda] = 'u'; - _foldCase[0xdb] = 'u'; - _foldCase[0xdd] = 'y'; - - _foldCase[0xe0] = 'a'; - _foldCase[0xe1] = 'a'; - _foldCase[0xe2] = 'a'; - _foldCase[0xe3] = 'a'; // a tilde - _foldCase[0xe7] = 'c'; - _foldCase[0xe8] = 'e'; - _foldCase[0xe9] = 'e'; - _foldCase[0xea] = 'e'; - _foldCase[0xeb] = 'e'; - _foldCase[0xec] = 'i'; // i grave - _foldCase[0xed] = 'i'; - _foldCase[0xee] = 'i'; - _foldCase[0xef] = 'i'; - _foldCase[0xf1] = 'n'; - _foldCase[0xf2] = 'o'; - _foldCase[0xf3] = 'o'; - _foldCase[0xf4] = 'o'; - _foldCase[0xf5] = 'o'; - _foldCase[0xf9] = 'u'; - _foldCase[0xfa] = 'u'; - _foldCase[0xfb] = 'u'; - _foldCase[0xfd] = 'y'; - _foldCase[0xff] = 'y'; - _foldCase[0x102] = 'a'; - _foldCase[0x103] = 'a'; - _foldCase[0x110] = 'd'; - _foldCase[0x111] = 'd'; - _foldCase[0x128] = 'i'; - _foldCase[0x129] = 'i'; - _foldCase[0x178] = 'y'; - _foldCase[0x1a0] = 'o'; - _foldCase[0x1a1] = 'o'; - _foldCase[0x1af] = 'u'; - _foldCase[0x1b0] = 'u'; - - // Superscript spacing modifiers - _foldCase[0x2b0] = 'h'; - _foldCase[0x2b1] = 0x266; - _foldCase[0x2b2] = 'j'; - _foldCase[0x2b3] = 'r'; - _foldCase[0x2b4] = 0x279; - _foldCase[0x2b5] = 0x27b; - _foldCase[0x2b6] = 0x281; - _foldCase[0x2b7] = 'w'; - _foldCase[0x2b8] = 'y'; - _foldCase[0x2e0] = 0x263; - _foldCase[0x2e1] = 'l'; - _foldCase[0x2e2] = 's'; - _foldCase[0x2e3] = 'x'; - _foldCase[0x2e4] = 0x295; - - _keepCase[0xc0] = 'A'; - _keepCase[0xc1] = 'A'; - _keepCase[0xc2] = 'A'; - _keepCase[0xc3] = 'A'; // A tilde - _keepCase[0xc7] = 'C'; - _keepCase[0xc8] = 'E'; - _keepCase[0xc9] = 'E'; - _keepCase[0xca] = 'E'; - _keepCase[0xcb] = 'E'; - _keepCase[0xcc] = 'I'; // I grave - _keepCase[0xcd] = 'I'; - _keepCase[0xce] = 'I'; - _keepCase[0xcf] = 'I'; - _keepCase[0xd1] = 'N'; - _keepCase[0xd2] = 'O'; - _keepCase[0xd3] = 'O'; - _keepCase[0xd4] = 'O'; - _keepCase[0xd5] = 'O'; - _keepCase[0xd9] = 'U'; - _keepCase[0xda] = 'U'; - _keepCase[0xdb] = 'U'; - _keepCase[0xdd] = 'Y'; - - _keepCase[0xe0] = 'a'; - _keepCase[0xe1] = 'a'; - _keepCase[0xe2] = 'a'; - _keepCase[0xe3] = 'a'; // a tilde - _keepCase[0xe7] = 'c'; - _keepCase[0xe8] = 'e'; - _keepCase[0xe9] = 'e'; - _keepCase[0xea] = 'e'; - _keepCase[0xeb] = 'e'; - _keepCase[0xec] = 'i'; // i grave - _keepCase[0xed] = 'i'; - _keepCase[0xee] = 'i'; - _keepCase[0xef] = 'i'; - _keepCase[0xf1] = 'n'; - _keepCase[0xf2] = 'o'; - _keepCase[0xf3] = 'o'; - _keepCase[0xf4] = 'o'; - _keepCase[0xf5] = 'o'; - _keepCase[0xf9] = 'u'; - _keepCase[0xfa] = 'u'; - _keepCase[0xfb] = 'u'; - _keepCase[0xfd] = 'y'; - _keepCase[0xff] = 'y'; - - _keepCase[0x102] = 'A'; - _keepCase[0x103] = 'a'; - _keepCase[0x110] = 'D'; - _keepCase[0x111] = 'd'; - _keepCase[0x128] = 'I'; - _keepCase[0x129] = 'i'; - _keepCase[0x178] = 'Y'; - _keepCase[0x1a0] = 'O'; - _keepCase[0x1a1] = 'o'; - _keepCase[0x1af] = 'U'; - _keepCase[0x1b0] = 'u'; - - // Superscript spacing modifiers - _foldCase[0x2b0] = 'h'; - _foldCase[0x2b1] = 0x266; - _foldCase[0x2b2] = 'j'; - _foldCase[0x2b3] = 'r'; - _foldCase[0x2b4] = 0x279; - _foldCase[0x2b5] = 0x27b; - _foldCase[0x2b6] = 0x281; - _foldCase[0x2b7] = 'w'; - _foldCase[0x2b8] = 'y'; - _foldCase[0x2e0] = 0x263; - _foldCase[0x2e1] = 'l'; - _foldCase[0x2e2] = 's'; - _foldCase[0x2e3] = 'x'; - _foldCase[0x2e4] = 0x295; - - // Deaccenting-table for Ascii Extended Additional - _foldCaseHighAscii[0x1ea0 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea1 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea2 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea3 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea4 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea5 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea6 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea7 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea8 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ea9 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eaa - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eab - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eac - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1ead - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eae - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eaf - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb0 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb1 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb2 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb3 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb4 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb5 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb6 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb7 - 0x1e00] = 'a'; - _foldCaseHighAscii[0x1eb8 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1eb9 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1eba - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ebb - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ebc - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ebd - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ebe - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ebf - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec0 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec1 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec2 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec3 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec4 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec5 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec6 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec7 - 0x1e00] = 'e'; - _foldCaseHighAscii[0x1ec8 - 0x1e00] = 'i'; - _foldCaseHighAscii[0x1ec9 - 0x1e00] = 'i'; - _foldCaseHighAscii[0x1eca - 0x1e00] = 'i'; - _foldCaseHighAscii[0x1ecb - 0x1e00] = 'i'; - _foldCaseHighAscii[0x1ecc - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ecd - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ece - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ecf - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed0 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed1 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed2 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed3 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed4 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed5 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed6 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed7 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed8 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ed9 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1eda - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1edb - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1edc - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1edd - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ede - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1edf - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ee0 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ee1 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ee2 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ee3 - 0x1e00] = 'o'; - _foldCaseHighAscii[0x1ee4 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ee5 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ee6 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ee7 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ee8 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ee9 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1eea - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1eeb - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1eec - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1eed - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1eee - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1eef - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ef0 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ef1 - 0x1e00] = 'u'; - _foldCaseHighAscii[0x1ef2 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef3 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef4 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef5 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef6 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef7 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef8 - 0x1e00] = 'y'; - _foldCaseHighAscii[0x1ef9 - 0x1e00] = 'y'; - - _keepCaseHighAscii[0x1ea0 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1ea1 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1ea2 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1ea3 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1ea4 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1ea5 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1ea6 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1ea7 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1ea8 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1ea9 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eaa - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1eab - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eac - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1ead - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eae - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1eaf - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eb0 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1eb1 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eb2 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1eb3 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eb4 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1eb5 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eb6 - 0x1e00] = 'A'; - _keepCaseHighAscii[0x1eb7 - 0x1e00] = 'a'; - _keepCaseHighAscii[0x1eb8 - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1eb9 - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1eba - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ebb - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ebc - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ebd - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ebe - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ebf - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ec0 - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ec1 - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ec2 - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ec3 - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ec4 - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ec5 - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ec6 - 0x1e00] = 'E'; - _keepCaseHighAscii[0x1ec7 - 0x1e00] = 'e'; - _keepCaseHighAscii[0x1ec8 - 0x1e00] = 'I'; - _keepCaseHighAscii[0x1ec9 - 0x1e00] = 'i'; - _keepCaseHighAscii[0x1eca - 0x1e00] = 'I'; - _keepCaseHighAscii[0x1ecb - 0x1e00] = 'i'; - _keepCaseHighAscii[0x1ecc - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ecd - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ece - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ecf - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ed0 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ed1 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ed2 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ed3 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ed4 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ed5 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ed6 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ed7 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ed8 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ed9 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1eda - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1edb - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1edc - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1edd - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ede - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1edf - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ee0 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ee1 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ee2 - 0x1e00] = 'O'; - _keepCaseHighAscii[0x1ee3 - 0x1e00] = 'o'; - _keepCaseHighAscii[0x1ee4 - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1ee5 - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1ee6 - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1ee7 - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1ee8 - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1ee9 - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1eea - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1eeb - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1eec - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1eed - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1eee - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1eef - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1ef0 - 0x1e00] = 'U'; - _keepCaseHighAscii[0x1ef1 - 0x1e00] = 'u'; - _keepCaseHighAscii[0x1ef2 - 0x1e00] = 'Y'; - _keepCaseHighAscii[0x1ef3 - 0x1e00] = 'y'; - _keepCaseHighAscii[0x1ef4 - 0x1e00] = 'Y'; - _keepCaseHighAscii[0x1ef5 - 0x1e00] = 'y'; - _keepCaseHighAscii[0x1ef6 - 0x1e00] = 'Y'; - _keepCaseHighAscii[0x1ef7 - 0x1e00] = 'y'; - _keepCaseHighAscii[0x1ef8 - 0x1e00] = 'Y'; - _keepCaseHighAscii[0x1ef9 - 0x1e00] = 'y'; - - } - - // Base case hiragana - hiragana ID - for (i = 0; i < 96; i++) { - _kanaMap[i] = 0x3040 + i; - } - - // Modify some hiragana - hiragana - if (_doSmallToNormalKana) { - // A I U E O YA YU YO WA, and TSU (previously we did not convert TSU) - _kanaMap[0x3041 - 0x3040] = 0x3042; - _kanaMap[0x3043 - 0x3040] = 0x3044; - _kanaMap[0x3045 - 0x3040] = 0x3046; - _kanaMap[0x3047 - 0x3040] = 0x3048; - _kanaMap[0x3049 - 0x3040] = 0x304A; - _kanaMap[0x3063 - 0x3040] = 0x3064; - _kanaMap[0x3083 - 0x3040] = 0x3084; - _kanaMap[0x3085 - 0x3040] = 0x3086; - _kanaMap[0x3087 - 0x3040] = 0x3088; - _kanaMap[0x308E - 0x3040] = 0x308F; - } - - if (_doKatakanaToHiragana) { - // base katakana to hiragana - for (i = 96; i < 192; i++) { - _kanaMap[i] = 0x3040 + i - 0x60; - } - - // modify some katakana - hiragana - - // 0x30A0 -> id - _kanaMap[0x30A0 - 0x3040] = 0x30A0; - // 0x30F7 to 0x30FC -> id - _kanaMap[0x30F7 - 0x3040] = 0x30F7; - _kanaMap[0x30F8 - 0x3040] = 0x30F8; - _kanaMap[0x30F9 - 0x3040] = 0x30F9; - _kanaMap[0x30FA - 0x3040] = 0x30FA; - _kanaMap[0x30FB - 0x3040] = 0x30FB; - _kanaMap[0x30FC - 0x3040] = 0x30FC; - // 0x30FF -> id - _kanaMap[0x30FF - 0x3040] = 0x30FF; - - if (_doSmallToNormalKana) { - // A I U E O YA YU YO WA, and TSU (previously we did not convert TSU) - _kanaMap[0x30A1 - 0x3040] = 0x3042; - _kanaMap[0x30A3 - 0x3040] = 0x3044; - _kanaMap[0x30A5 - 0x3040] = 0x3046; - _kanaMap[0x30A7 - 0x3040] = 0x3048; - _kanaMap[0x30A9 - 0x3040] = 0x304A; - _kanaMap[0x30C3 - 0x3040] = 0x30C4; - _kanaMap[0x30E3 - 0x3040] = 0x3084; - _kanaMap[0x30E5 - 0x3040] = 0x3086; - _kanaMap[0x30E7 - 0x3040] = 0x3088; - _kanaMap[0x30EE - 0x3040] = 0x308F; - // KA KE - _kanaMap[0x30F5 - 0x3040] = 0x304B; - _kanaMap[0x30F6 - 0x3040] = 0x3051; - } else { // !_doSmallToNormalKana - // A I U E O YA YU YO WA, not TSU is normal katakana - hiragana - // KA KE; No small hiragana exists, so id - _kanaMap[0x30F5 - 0x3040] = 0x30F5; - _kanaMap[0x30F6 - 0x3040] = 0x30F6; - } - } else { // !_doKatakanaToHiragana - // katakana - katakana ID - for (i = 96; i < 192; i++) { - _kanaMap[i] = 0x3040 + i; - } - - // modify some katakana - katakana - if (_doSmallToNormalKana) { - // A I U E O YA YU YO WA, not TSU - _kanaMap[0x30A1 - 0x3040] = 0x30A2; - _kanaMap[0x30A3 - 0x3040] = 0x30A4; - _kanaMap[0x30A5 - 0x3040] = 0x30A6; - _kanaMap[0x30A7 - 0x3040] = 0x30A8; - _kanaMap[0x30A9 - 0x3040] = 0x30AA; - _kanaMap[0x30E3 - 0x3040] = 0x30E4; - _kanaMap[0x30E5 - 0x3040] = 0x30E6; - _kanaMap[0x30E7 - 0x3040] = 0x30E8; - _kanaMap[0x30EE - 0x3040] = 0x30EF; - // KA KE - _kanaMap[0x30F5 - 0x3040] = 0x30AB; - _kanaMap[0x30F6 - 0x3040] = 0x30B1; - } - } - - - - // Fullwidth ASCII - for (i = 0; i < 0x21; i++) - _halfwidth_fullwidthMap[i] = 0x20 + i; - for (i = 0x21; i < 0x3B; i++) // full uppercase to half lowercase - _halfwidth_fullwidthMap[i] = 0x40 + i; - for (i = 0x3B; i < 0x5F; i++) - _halfwidth_fullwidthMap[i] = 0x20 + i; - // 0xFF00, 0xFF5F -> id - _halfwidth_fullwidthMap[0x00] = 0xFF00; - _halfwidth_fullwidthMap[0x5F] = 0xFF5F; - - // Halfwidth CJK Punctuation - // 0xFF60 -> id - _halfwidth_fullwidthMap[0x60] = 0xFF60; - _halfwidth_fullwidthMap[0x61] = 0x3002; - _halfwidth_fullwidthMap[0x62] = 0x300C; - _halfwidth_fullwidthMap[0x63] = 0x300D; - _halfwidth_fullwidthMap[0x64] = 0x3001; - - // Halfwidth katakana (maps directly to hiragana) - - // Common cases for halfwidth katakana - _halfwidth_fullwidthMap[0x65] = 0x30FB; - - if (_doKatakanaToHiragana) { - _halfwidth_fullwidthMap[0x66] = 0x3092; - _halfwidth_fullwidthMap[0x6F] = 0x3063; - _halfwidth_fullwidthMap[0x70] = 0x30FC; - _halfwidth_fullwidthMap[0x71] = 0x3042; - _halfwidth_fullwidthMap[0x72] = 0x3044; - _halfwidth_fullwidthMap[0x73] = 0x3046; - _halfwidth_fullwidthMap[0x74] = 0x3048; - _halfwidth_fullwidthMap[0x75] = 0x304A; - _halfwidth_fullwidthMap[0x76] = 0x304B; - _halfwidth_fullwidthMap[0x77] = 0x304D; - _halfwidth_fullwidthMap[0x78] = 0x304F; - _halfwidth_fullwidthMap[0x79] = 0x3051; - _halfwidth_fullwidthMap[0x7A] = 0x3053; - _halfwidth_fullwidthMap[0x7B] = 0x3055; - _halfwidth_fullwidthMap[0x7C] = 0x3057; - _halfwidth_fullwidthMap[0x7D] = 0x3059; - _halfwidth_fullwidthMap[0x7E] = 0x305B; - _halfwidth_fullwidthMap[0x7F] = 0x305D; - _halfwidth_fullwidthMap[0x80] = 0x305F; - _halfwidth_fullwidthMap[0x81] = 0x3061; - _halfwidth_fullwidthMap[0x82] = 0x3064; - _halfwidth_fullwidthMap[0x83] = 0x3066; - _halfwidth_fullwidthMap[0x84] = 0x3068; - _halfwidth_fullwidthMap[0x85] = 0x306A; - _halfwidth_fullwidthMap[0x86] = 0x306B; - _halfwidth_fullwidthMap[0x87] = 0x306C; - _halfwidth_fullwidthMap[0x88] = 0x306D; - _halfwidth_fullwidthMap[0x89] = 0x306E; - _halfwidth_fullwidthMap[0x8A] = 0x306F; - _halfwidth_fullwidthMap[0x8B] = 0x3072; - _halfwidth_fullwidthMap[0x8C] = 0x3075; - _halfwidth_fullwidthMap[0x8D] = 0x3078; - _halfwidth_fullwidthMap[0x8E] = 0x307B; - _halfwidth_fullwidthMap[0x8F] = 0x307E; - _halfwidth_fullwidthMap[0x90] = 0x307F; - _halfwidth_fullwidthMap[0x91] = 0x3080; - _halfwidth_fullwidthMap[0x92] = 0x3081; - _halfwidth_fullwidthMap[0x93] = 0x3082; - _halfwidth_fullwidthMap[0x94] = 0x3084; - _halfwidth_fullwidthMap[0x95] = 0x3086; - _halfwidth_fullwidthMap[0x96] = 0x3088; - _halfwidth_fullwidthMap[0x97] = 0x3089; - _halfwidth_fullwidthMap[0x98] = 0x308A; - _halfwidth_fullwidthMap[0x99] = 0x308B; - _halfwidth_fullwidthMap[0x9A] = 0x308C; - _halfwidth_fullwidthMap[0x9B] = 0x308D; - _halfwidth_fullwidthMap[0x9C] = 0x308F; - _halfwidth_fullwidthMap[0x9D] = 0x3093; - _halfwidth_fullwidthMap[0x9E] = 0x3099; - _halfwidth_fullwidthMap[0x9F] = 0x309A; - if (_doSmallToNormalKana) { - _halfwidth_fullwidthMap[0x67] = 0x3042; - _halfwidth_fullwidthMap[0x68] = 0x3044; - _halfwidth_fullwidthMap[0x69] = 0x3046; - _halfwidth_fullwidthMap[0x6A] = 0x3048; - _halfwidth_fullwidthMap[0x6B] = 0x304A; - _halfwidth_fullwidthMap[0x6C] = 0x3084; - _halfwidth_fullwidthMap[0x6D] = 0x3086; - _halfwidth_fullwidthMap[0x6E] = 0x3088; - } else { // !_doSmallToNormalKana - _halfwidth_fullwidthMap[0x67] = 0x3041; - _halfwidth_fullwidthMap[0x68] = 0x3043; - _halfwidth_fullwidthMap[0x69] = 0x3045; - _halfwidth_fullwidthMap[0x6A] = 0x3047; - _halfwidth_fullwidthMap[0x6B] = 0x3049; - _halfwidth_fullwidthMap[0x6C] = 0x3083; - _halfwidth_fullwidthMap[0x6D] = 0x3085; - _halfwidth_fullwidthMap[0x6E] = 0x3087; - } - } else { // !_doKatakanaToHiragana - _halfwidth_fullwidthMap[0x66] = 0x30F2; - _halfwidth_fullwidthMap[0x6F] = 0x30C3; - _halfwidth_fullwidthMap[0x70] = 0x30FC; - _halfwidth_fullwidthMap[0x71] = 0x30A2; - _halfwidth_fullwidthMap[0x72] = 0x30A4; - _halfwidth_fullwidthMap[0x73] = 0x30A6; - _halfwidth_fullwidthMap[0x74] = 0x30A8; - _halfwidth_fullwidthMap[0x75] = 0x30AA; - _halfwidth_fullwidthMap[0x76] = 0x30AB; - _halfwidth_fullwidthMap[0x77] = 0x30AD; - _halfwidth_fullwidthMap[0x78] = 0x30AF; - _halfwidth_fullwidthMap[0x79] = 0x30B1; - _halfwidth_fullwidthMap[0x7A] = 0x30B3; - _halfwidth_fullwidthMap[0x7B] = 0x30B5; - _halfwidth_fullwidthMap[0x7C] = 0x30B7; - _halfwidth_fullwidthMap[0x7D] = 0x30B9; - _halfwidth_fullwidthMap[0x7E] = 0x30BB; - _halfwidth_fullwidthMap[0x7F] = 0x30BD; - _halfwidth_fullwidthMap[0x80] = 0x30BF; - _halfwidth_fullwidthMap[0x81] = 0x30C1; - _halfwidth_fullwidthMap[0x82] = 0x30C4; - _halfwidth_fullwidthMap[0x83] = 0x30C6; - _halfwidth_fullwidthMap[0x84] = 0x30C8; - _halfwidth_fullwidthMap[0x85] = 0x30CA; - _halfwidth_fullwidthMap[0x86] = 0x30CB; - _halfwidth_fullwidthMap[0x87] = 0x30CC; - _halfwidth_fullwidthMap[0x88] = 0x30CD; - _halfwidth_fullwidthMap[0x89] = 0x30CE; - _halfwidth_fullwidthMap[0x8A] = 0x30CF; - _halfwidth_fullwidthMap[0x8B] = 0x30D2; - _halfwidth_fullwidthMap[0x8C] = 0x30D5; - _halfwidth_fullwidthMap[0x8D] = 0x30D8; - _halfwidth_fullwidthMap[0x8E] = 0x30DB; - _halfwidth_fullwidthMap[0x8F] = 0x30DE; - _halfwidth_fullwidthMap[0x90] = 0x30DF; - _halfwidth_fullwidthMap[0x91] = 0x30E0; - _halfwidth_fullwidthMap[0x92] = 0x30E1; - _halfwidth_fullwidthMap[0x93] = 0x30E2; - _halfwidth_fullwidthMap[0x94] = 0x30E4; - _halfwidth_fullwidthMap[0x95] = 0x30E6; - _halfwidth_fullwidthMap[0x96] = 0x30E8; - _halfwidth_fullwidthMap[0x97] = 0x30E9; - _halfwidth_fullwidthMap[0x98] = 0x30EA; - _halfwidth_fullwidthMap[0x99] = 0x30EB; - _halfwidth_fullwidthMap[0x9A] = 0x30EC; - _halfwidth_fullwidthMap[0x9B] = 0x30ED; - _halfwidth_fullwidthMap[0x9C] = 0x30EF; - _halfwidth_fullwidthMap[0x9D] = 0x30F3; - _halfwidth_fullwidthMap[0x9E] = 0x3099; - _halfwidth_fullwidthMap[0x9F] = 0x309A; - if (_doSmallToNormalKana) { - _halfwidth_fullwidthMap[0x67] = 0x30a2; - _halfwidth_fullwidthMap[0x68] = 0x30a4; - _halfwidth_fullwidthMap[0x69] = 0x30a6; - _halfwidth_fullwidthMap[0x6A] = 0x30a8; - _halfwidth_fullwidthMap[0x6B] = 0x30aA; - _halfwidth_fullwidthMap[0x6C] = 0x30e4; - _halfwidth_fullwidthMap[0x6D] = 0x30e6; - _halfwidth_fullwidthMap[0x6E] = 0x30e8; - } else { // !_doSmallToNormalKana - _halfwidth_fullwidthMap[0x67] = 0x30a1; - _halfwidth_fullwidthMap[0x68] = 0x30a3; - _halfwidth_fullwidthMap[0x69] = 0x30a5; - _halfwidth_fullwidthMap[0x6A] = 0x30a7; - _halfwidth_fullwidthMap[0x6B] = 0x30a9; - _halfwidth_fullwidthMap[0x6C] = 0x30e3; - _halfwidth_fullwidthMap[0x6D] = 0x30e5; - _halfwidth_fullwidthMap[0x6E] = 0x30e7; - } - } - - // Halfwidth Hangul - _halfwidth_fullwidthMap[0xA0] = 0x3164; - // fill in 0xFFA1 - 0xFFBE => 0x3131 - 0x314E - for (i = 0xA1; i < 0xBF; i++) - _halfwidth_fullwidthMap[i] = 0x3090 + i; - _halfwidth_fullwidthMap[0xBF] = 0xFFBF; - _halfwidth_fullwidthMap[0xC0] = 0xFFC0; - _halfwidth_fullwidthMap[0xC1] = 0xFFC1; - // fill in 0xFFC2 - 0xFFC7 => 0x314F - 0x3154 - for (i = 0xC2; i < 0xC8; i++) - _halfwidth_fullwidthMap[i] = 0x308D + i; - _halfwidth_fullwidthMap[0xC8] = 0xFFC8; - _halfwidth_fullwidthMap[0xC9] = 0xFFC9; - // fill in 0xFFCA - 0xFFCF => 0x3155 - 0x315A - for (i = 0xCA; i < 0xD0; i++) - _halfwidth_fullwidthMap[i] = 0x308B + i; - _halfwidth_fullwidthMap[0xD0] = 0xFFD0; - _halfwidth_fullwidthMap[0xD1] = 0xFFD1; - // fill in 0xFFD2 - 0xFFD7 => 0x315B - 0x3160 - for (i = 0xD2; i < 0xD8; i++) - _halfwidth_fullwidthMap[i] = 0x3089 + i; - _halfwidth_fullwidthMap[0xD8] = 0xFFD8; - _halfwidth_fullwidthMap[0xD9] = 0xFFD9; - // fill in 0xFFDA - 0xFFDC => 0x3161 - 0x3163 - for (i = 0xDA; i < 0xDD; i++) - _halfwidth_fullwidthMap[i] = 0x3087 + i; - - // Fullwidth symbols - _halfwidth_fullwidthMap[0xE0] = 0x00A2; - _halfwidth_fullwidthMap[0xE1] = 0x00A3; - _halfwidth_fullwidthMap[0xE2] = 0x00AC; - _halfwidth_fullwidthMap[0xE3] = 0x00AF; - _halfwidth_fullwidthMap[0xE4] = 0x00A6; - _halfwidth_fullwidthMap[0xE5] = 0x00A5; - _halfwidth_fullwidthMap[0xE6] = 0x20A9; - - // 0xFFE7 -> id - _halfwidth_fullwidthMap[0xE7] = 0xFFE7; - - // Halfwidth symbols - _halfwidth_fullwidthMap[0xE8] = 0x2502; - _halfwidth_fullwidthMap[0xE9] = 0x2190; - _halfwidth_fullwidthMap[0xEA] = 0x2191; - _halfwidth_fullwidthMap[0xEB] = 0x2192; - _halfwidth_fullwidthMap[0xEC] = 0x2193; - _halfwidth_fullwidthMap[0xED] = 0x25A0; - _halfwidth_fullwidthMap[0xEE] = 0x25CB; - - // 0xFFEF -> id - _halfwidth_fullwidthMap[0xEF] = 0xFFEF; - - - // - // DONE - // - _isInitialized = true; - } - } -} - -Fast_NormalizeWordFolder::Fast_NormalizeWordFolder() -{ - Initialize(); -} - - -Fast_NormalizeWordFolder::~Fast_NormalizeWordFolder(void) -{ -} - -size_t -Fast_NormalizeWordFolder::FoldedSizeAsUTF8(const char *word) const -{ - ucs4_t c; - size_t res; - const unsigned char *uword; - - res = 0; - uword = reinterpret_cast<const unsigned char *>(word); - c = Fast_UnicodeUtil::GetUTF8Char(uword); - while (c != 0) { - if (c != Fast_UnicodeUtil::_BadUTF8Char) { - const char *repl = ReplacementString(c); - if (repl != NULL) { - res += strlen(repl); - } else { - c = ToFold(c); - res += Fast_UnicodeUtil::utf8clen(c); - } - } - c = Fast_UnicodeUtil::GetUTF8Char(uword); - } - return res; -} - - -char * -Fast_NormalizeWordFolder::FoldUTF8WordToUTF8Quick(char *wordbufpos, - const char *word) - const -{ - ucs4_t c; - const unsigned char *uword; - - uword = reinterpret_cast<const unsigned char *>(word); - c = Fast_UnicodeUtil::GetUTF8Char(uword); - while (c != 0) { - if (c != Fast_UnicodeUtil::_BadUTF8Char) { - const char *repl = ReplacementString(c); - if (repl != NULL) { - size_t repllen = strlen(repl); - if (repllen > 0) - memcpy(wordbufpos, repl, repllen); - wordbufpos += repllen; - } else { - c = ToFold(c); - wordbufpos = Fast_UnicodeUtil::utf8cput(wordbufpos, c); - } - } - c = Fast_UnicodeUtil::GetUTF8Char(uword); - } - return wordbufpos; -} - -const char* -Fast_NormalizeWordFolder::Tokenize(const char *buf, - const char *bufend, - char *dstbuf, - char *dstbufend, - const char*& origstart, - size_t& tokenlen) const -{ - - ucs4_t c = 0; - const unsigned char *p; - char *q = NULL; - char *eq = NULL; - const unsigned char *ep; - p = reinterpret_cast<const unsigned char *>(buf); - ep = reinterpret_cast<const unsigned char *>(bufend); - - // Skip characters between words - for (;;) { - if (p >= ep) { // End of input buffer, no more words - *dstbuf = 0; - return reinterpret_cast<const char *>(p); - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (_isWord[c]) - { - origstart = reinterpret_cast<const char *>(p) - 1; - break; - } - } else { - const unsigned char* prev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (IsWordCharOrIA(c)) - { - origstart = reinterpret_cast<const char *>(prev_p); - break; - } - } - } - - // Start saving word. - q = dstbuf; - eq = dstbufend - 6; // Make room for long UTF8 char and NUL - // Doesn't check for space for the first char, assumes that - // word buffer is at least 13 characters - if (c < 128) { // Common case, ASCII - *q++ = _foldCase[c]; - } else { - const char *repl = ReplacementString(c); - if (repl != NULL) { - size_t repllen = strlen(repl); - if (repllen > 0) - memcpy(q, repl, repllen); - q += repllen; - } else { - c = ToFold(c); - q = Fast_UnicodeUtil::utf8cput(q, c); - } - } - - // Special case for interlinear annotation - if (c == 0xFFF9) { // ANCHOR - // Collect up to and including terminator - for(;;) { - if (p >= ep) { - c = 0; - break; - } - if (*p < 128) { // Note, no exit on plain ASCII - c = *p++; - *q++ = c; - if (q >= eq) { // Junk rest of annotation block - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - } else { - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (c == 0xFFFB) { - break; // out of junking loop - } - } - } - break; // out of annotation block processing - } - } else { - c = Fast_UnicodeUtil::GetUTF8Char(p); - q = Fast_UnicodeUtil::utf8cput(q, c); - if (c == 0xFFFB) { // TERMINATOR => Exit condition - break; - } - if (q >= eq) { // Junk rest of word - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - } else { - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (c == 0xFFFB) { - break; - } - } - } - break; - } - } - } - } else - - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (!_isWord[c]) - { - p--; - break; - } - *q++ = _foldCase[c]; - if (q >= eq) { // Junk rest of word - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (!_isWord[c]) - { - p--; - break; - } - } else { - const unsigned char* prev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (!Fast_UnicodeUtil::IsWordChar(c)) - { - p = prev_p; - break; - } - } - } - break; - } - } else { - const unsigned char* prev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (!Fast_UnicodeUtil::IsWordChar(c)) - { - p = prev_p; - break; - } - const char *repl = ReplacementString(c); - if (repl != NULL) { - size_t repllen = strlen(repl); - if (repllen > 0) - memcpy(q, repl, repllen); - q += repllen; - } else { - c = ToFold(c); - q = Fast_UnicodeUtil::utf8cput(q, c); - } - if (q >= eq) { // Junk rest of word - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (!_isWord[c]) - { - p--; - break; - } - } else { - const unsigned char* xprev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (!Fast_UnicodeUtil::IsWordChar(c)) - { - p = xprev_p; - break; - } - } - } - break; - } - } - } - *q = 0; - tokenlen = q - dstbuf; - return reinterpret_cast<const char *>(p); -} - - - -const char* -Fast_NormalizeWordFolder::UCS4Tokenize(const char *buf, - const char *bufend, - ucs4_t *dstbuf, - ucs4_t *dstbufend, - const char*& origstart, - size_t& tokenlen) const -{ - return Tokenize(buf, bufend, dstbuf, dstbufend, origstart, tokenlen); -} - -const char* -Fast_NormalizeWordFolder::Tokenize(const char *buf, - const char *bufend, - ucs4_t *dstbuf, - ucs4_t *dstbufend, - const char*& origstart, - size_t& tokenlen) const -{ - - ucs4_t c = 0; - const unsigned char *p; - ucs4_t *q = NULL; - ucs4_t *eq = NULL; - const unsigned char *ep; - p = reinterpret_cast<const unsigned char *>(buf); - ep = reinterpret_cast<const unsigned char *>(bufend); - - // Skip characters between words - for (;;) { - if (p >= ep) { // End of input buffer, no more words - *dstbuf = 0; - return reinterpret_cast<const char *>(p); - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (_isWord[c]) - { - origstart = reinterpret_cast<const char *>(p) - 1; - break; - } - } else { - const unsigned char* prev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (IsWordCharOrIA(c)) - { - origstart = reinterpret_cast<const char *>(prev_p); - break; - } - } - } - - // Start saving word. - q = dstbuf; - eq = dstbufend - 3; // Make room for UCS4 char replacement string and NUL - // Doesn't check for space for the first char, assumes that - // word buffer is at least 13 characters - if (c < 128) { // Common case, ASCII - *q++ = _foldCase[c]; - } else { - const char *repl = ReplacementString(c); - if (repl != NULL) { - size_t repllen = strlen(repl); - if (repllen > 0) - q = Fast_UnicodeUtil::ucs4copy(q,repl); - } else { - c = ToFold(c); - *q++ = c; - } - } - - // Special case for interlinear annotation - if (c == 0xFFF9) { // ANCHOR - // Collect up to and including terminator - for(;;) { - if (p >= ep) { - c = 0; - break; - } - if (*p < 128) { // Note, no exit on plain ASCII - c = *p++; - *q++ = c; - if (q >= eq) { // Junk rest of annotation block - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - } else { - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (c == 0xFFFB) { - break; // out of junking loop - } - } - } - break; // out of annotation block processing - } - } else { - c = Fast_UnicodeUtil::GetUTF8Char(p); - *q++ = c; - if (c == 0xFFFB) { // TERMINATOR => Exit condition - break; - } - if (q >= eq) { // Junk rest of word - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - } else { - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (c == 0xFFFB) { - break; - } - } - } - break; - } - } - } - } else - - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (!_isWord[c]) - { - p--; - break; - } - *q++ = _foldCase[c]; - if (q >= eq) { // Junk rest of word - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (!_isWord[c]) - { - p--; - break; - } - } else { - const unsigned char* prev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (!Fast_UnicodeUtil::IsWordChar(c)) - { - p = prev_p; - break; - } - } - } - break; - } - } else { - const unsigned char* prev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (!Fast_UnicodeUtil::IsWordChar(c)) - { - p = prev_p; - break; - } - const char *repl = ReplacementString(c); - if (repl != NULL) { - size_t repllen = strlen(repl); - if (repllen > 0) - q = Fast_UnicodeUtil::ucs4copy(q,repl); - } else { - c = ToFold(c); - *q++ = c; - } - if (q >= eq) { // Junk rest of word - for (;;) { - if (p >= ep) { // End of input buffer - c = 0; - break; - } - if (*p < 128) { // Common case, ASCII - c = *p++; - if (!_isWord[c]) - { - p--; - break; - } - } else { - const unsigned char* xprev_p = p; - c = Fast_UnicodeUtil::GetUTF8Char(p); - if (!Fast_UnicodeUtil::IsWordChar(c)) - { - p = xprev_p; - break; - } - } - } - break; - } - } - } - *q = 0; - tokenlen = q - dstbuf; - return reinterpret_cast<const char *>(p); -} diff --git a/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp b/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp deleted file mode 100644 index 4120e574233..00000000000 --- a/fastlib/src/vespa/fastlib/text/tests/characterclasstest.cpp +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "characterclasstest.h" - -int character_class_test_app::Main() -{ - character_class_test t("Test for the character_class"); - t.SetStream(&std::cout); - t.Run(); - return t.Report(); -} - -FASTOS_MAIN(character_class_test_app) diff --git a/fastlib/src/vespa/fastlib/text/tests/unicodeutiltest.cpp b/fastlib/src/vespa/fastlib/text/tests/unicodeutiltest.cpp deleted file mode 100644 index 9e7fa176a8c..00000000000 --- a/fastlib/src/vespa/fastlib/text/tests/unicodeutiltest.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "unicodeutiltest.h" - -int main(int, char **) { - UnicodeUtilTest t; - t.SetStream(&std::cout); - t.Run(); - return t.Report(); -} diff --git a/fastlib/src/vespa/fastlib/text/tests/unicodeutiltest.h b/fastlib/src/vespa/fastlib/text/tests/unicodeutiltest.h deleted file mode 100644 index da7bae6d313..00000000000 --- a/fastlib/src/vespa/fastlib/text/tests/unicodeutiltest.h +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastlib/testsuite/test.h> -#include <vespa/fastlib/text/unicodeutil.h> - -class UnicodeUtilTest : public Test -{ - bool GetUTF8Char_WrongInput() { - const char *testdata = "ab\xF8"; - - ucs4_t the_char = 0; - - const unsigned char *src = reinterpret_cast<const unsigned char *>(testdata); - while (*src != 0) { - the_char = Fast_UnicodeUtil::GetUTF8Char(src); - // fprintf(stderr, "GetUTF8Char_WrongInput(): the_char = U+%04X\n", the_char); - } - return (the_char == Fast_UnicodeUtil::_BadUTF8Char); - } - bool IsTerminalPunctuationChar(char ch, bool b) { - if (Fast_UnicodeUtil::IsTerminalPunctuationChar(ch) != b) { - printf("expected char '%c' %s terminal punctuation char\n", ch, b ? "to be" : "not to be"); - return false; - } - return true; - } - - bool IsTerminalPunctuationChar() { - // test a small selection - bool retval = true; - retval &= IsTerminalPunctuationChar('!', true); - retval &= IsTerminalPunctuationChar(',', true); - retval &= IsTerminalPunctuationChar('.', true); - retval &= IsTerminalPunctuationChar(':', true); - retval &= IsTerminalPunctuationChar(';', true); - retval &= IsTerminalPunctuationChar(' ', false); - retval &= IsTerminalPunctuationChar('a', false); - retval &= IsTerminalPunctuationChar('A', false); - return retval; - } - -public: - void Run() override { - // do the tests - _test(GetUTF8Char_WrongInput()); - _test(IsTerminalPunctuationChar()); - } -}; diff --git a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp deleted file mode 100644 index ea254b3d114..00000000000 --- a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.cpp +++ /dev/null @@ -1,9 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "wordfolderstest.h" - -int main(int, char **) { - WordFoldersTest t; - t.SetStream(&std::cout); - t.Run(); - return t.Report(); -} diff --git a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h b/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h deleted file mode 100644 index b82415a9822..00000000000 --- a/fastlib/src/vespa/fastlib/text/tests/wordfolderstest.h +++ /dev/null @@ -1,127 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/fastlib/testsuite/test.h> -#include <vespa/fastlib/text/normwordfolder.h> -#include <memory> -#include <cstring> - -class WordFoldersTest : public Test -{ - bool NormalizeWordFolderConstruction() { - Fast_NormalizeWordFolder::Setup( - Fast_NormalizeWordFolder::DO_ACCENT_REMOVAL - | Fast_NormalizeWordFolder::DO_KATAKANA_TO_HIRAGANA - | Fast_NormalizeWordFolder::DO_SMALL_TO_NORMAL_KANA - | Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION - | Fast_NormalizeWordFolder::DO_LIGATURE_SUBSTITUTION - | Fast_NormalizeWordFolder::DO_MULTICHAR_EXPANSION); - - Fast_NormalizeWordFolder *nwf = new Fast_NormalizeWordFolder(); - delete nwf; - - return true; - } - - bool TokenizeAnnotatedBuffer() { - Fast_NormalizeWordFolder *nwf = new Fast_NormalizeWordFolder(); - const char *testinput = "This is a " - "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB" - " superduperextrafeaturecoolandlongplainword fun " - "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" - "world wide web extra long annotation block" "\xEF\xBF\xBB" - " test\nIt is cool.\n"; - const char *correct[] = { - "this", "is", "a", - "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB", - "superduperextrafeaturecool", "fun", - "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" "world wide web ex", - "test", "it", "is", "cool" }; - const char *teststart = testinput; - const char *testend = testinput + strlen(testinput); - char destbuf[32]; - char *destbufend = destbuf + 32; - const char *origstart = testinput; - size_t tokenlen = 0; - - int tokencounter = 0; - bool success = true; - while ( - (teststart - = nwf->Tokenize(teststart, testend, - destbuf, destbufend, - origstart, tokenlen)) < testend) { - // printf("found: %s, correct: %s\n", destbuf, correct[tokencounter]); - success &= strcmp(destbuf, correct[tokencounter++]) == 0; - } - - delete nwf; - - return success; - } - - bool TokenizeAnnotatedUCS4Buffer() { - Fast_NormalizeWordFolder *nwf = new Fast_NormalizeWordFolder(); - const char *testinput = "This is a " - "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB" - " superduperextrafeaturecoolandlongplainword fun " - "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" - "world wide web extra long annotation block" "\xEF\xBF\xBB" - " test\nIt is cool.\n"; - const char *correct[] = { - "this", "is", "a", - "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB", - "superduperextrafeaturecooland", "fun", - "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" "world wide web extra lon", - "test", "it", "is", "cool" }; - - const char *teststart = testinput; - const char *testend = testinput + strlen(testinput); - ucs4_t destbuf[32]; - ucs4_t *destbufend = destbuf + 32; - - const char *origstart = testinput; - size_t tokenlen = 0; - - int tokencounter = 0; - bool success = true; - while ( - (teststart - = nwf->UCS4Tokenize(teststart, testend, - destbuf, destbufend, - origstart, tokenlen)) < testend) { - success &= Fast_UnicodeUtil::utf8cmp(correct[tokencounter++], destbuf) == 0; - } - - delete nwf; - - return success; - } - - bool AccentRemovalTest() { - // Note last encoded characters encoded as octets to avoid interpreting following letters after xNN as part of the encoding of the character - // See http://en.cppreference.com/w/cpp/language/escape - auto freefunction = [] (char * ptr) { free(ptr); }; - auto input = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\xBF\xC0\xC1\xC2\xC3\xC4\xC5\xC6\xC7\xC8\xC9\xCA\xCB\xCC\xCD\xCE\xCF\xD0\xD1\xD2\xD3\xD4\xD5\xD6\xD7\xD8\xD9\xDA\xDB\xDC\xDD\xDE\xDF\xE0\xE1\xE2\xE3\xE4\xE5\xE6\xE7\xE8\xE9\xEA\xEB\xEC\xED\xEE\xEF\xF0\xF1\xF2\xF3\xF4\xF5\xF6\xF7\xF8\xF9\xFA\xFB\xFC\xFD\xFE\x70\xFE\x21"), - freefunction); - auto yelloutput = std::unique_ptr<char, decltype(freefunction)>(Fast_UnicodeUtil::strdupLAT1("\xA1\xA2\xA3\xA4\xA5\xA6\xA7\xA8\xA9\xAA\xAB\xAC\xAD\xAE\xAF\xB0\xB1\xB2\xB3\xB4\xB5\xB6\xB7\xB8\xB9\xBA\xBB\xBC\xBD\xBE\277AAAAAEAAAECEEEEIIIIDNOOOOOE\327OEUUUUEYTHssaaaaaeaaaeceeeeiiiidnoooooe\367oeuuuueythpth!"), - freefunction); - Fast_NormalizeWordFolder wordfolder; - int len = wordfolder.FoldedSizeAsUTF8(input.get()); - auto fastliboutput = std::unique_ptr<char[]>(new char[len + 1]); - wordfolder.FoldUTF8WordToUTF8Quick(fastliboutput.get(), input.get()); - fastliboutput[len] = '\0'; - printf("\n%s\n", yelloutput.get()); - printf("%s\n", fastliboutput.get()); - return strcasecmp(yelloutput.get(), fastliboutput.get()) == 0; - } - - -public: - - void Run() override { - // do the tests - _test(NormalizeWordFolderConstruction()); - _test(TokenizeAnnotatedBuffer()); - _test(TokenizeAnnotatedUCS4Buffer()); - _test(AccentRemovalTest()); - } -}; diff --git a/fastlib/src/vespa/fastlib/text/wordfolder.cpp b/fastlib/src/vespa/fastlib/text/wordfolder.cpp deleted file mode 100644 index e0b1aba9356..00000000000 --- a/fastlib/src/vespa/fastlib/text/wordfolder.cpp +++ /dev/null @@ -1,7 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "wordfolder.h" - -Fast_WordFolder::~Fast_WordFolder() -{ -} diff --git a/fastlib/src/vespa/fastlib/util/.gitignore b/fastlib/src/vespa/fastlib/util/.gitignore deleted file mode 100644 index da7183fc8af..00000000000 --- a/fastlib/src/vespa/fastlib/util/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -*.So -*.a -*.ilk -*.lib -*.o -*.obj -*.pdb -.cvsignore -.depend -.pure -Debug -Makefile -SunWS_cache -extcase -extprop diff --git a/fastlib/src/vespa/fastlib/util/testproject.el b/fastlib/src/vespa/fastlib/util/testproject.el deleted file mode 100644 index 94bbe60394d..00000000000 --- a/fastlib/src/vespa/fastlib/util/testproject.el +++ /dev/null @@ -1,20 +0,0 @@ -;; Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -;; testproject.el - -;; Local configurations for the cpptest Emacs unit-test framework - -;; Author: Nils Sandøy <nils.sandoy@fast.no> - -(message "Setting local test configuration") - -(setq cppt-test-dir "tests") - -(setq cppt-use-underscore-p t) - -(setq cppt-extra-object-files '()) - -;; We have no parameters to our test executables -(setq cppt-test-parameters "") - -;; Use -d to turn on debug mode -(setq cppt-test-dbflags "-d") diff --git a/fastlib/src/vespa/fastlib/util/tests/.gitignore b/fastlib/src/vespa/fastlib/util/tests/.gitignore deleted file mode 100644 index c43f5961bad..00000000000 --- a/fastlib/src/vespa/fastlib/util/tests/.gitignore +++ /dev/null @@ -1,42 +0,0 @@ -*.So -*.a -*.ilk -*.o -*.obj -*.pdb -.cvsignore -.depend -.pure -Debug -Makefile -SunWS_cache -arraytest -bagtest -base64test -bitstest -bloomfilter.* -bloomfiltertest -bobhashtest -crc32test -datetimetest -debugtest -fixedarraytest -flisttest -genkeyhashmaptest -hashmaptest -hashtabletest -refcounttest -resourcestatstest -ringbuffertest -scopeguardtest -singletontest -stacktest -stringbuffertest -stringtest -stringutilitytest -trietest -utiltestsuite -wildcard_match_test -fastlib_wildcard_match_test_app -fastlib_bagtest_app -fastlib_base64test_app diff --git a/fastlib/src/vespa/fastlib/util/tests/CMakeLists.txt b/fastlib/src/vespa/fastlib/util/tests/CMakeLists.txt deleted file mode 100644 index cf655b548e5..00000000000 --- a/fastlib/src/vespa/fastlib/util/tests/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(fastlib_wildcard_match_test_app TEST - SOURCES - wildcard_match_test.cpp - DEPENDS - fastlib_fast_testsuite -) -vespa_add_test(NAME fastlib_wildcard_match_test_app NO_VALGRIND COMMAND fastlib_wildcard_match_test_app) diff --git a/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp b/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp deleted file mode 100644 index a63a6943fc6..00000000000 --- a/fastlib/src/vespa/fastlib/util/tests/wildcard_match_test.cpp +++ /dev/null @@ -1,47 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <iostream> -#include "../wildcard_match.h" - -namespace -{ -template<typename T> -bool Test(const T* word, const T* pattern, bool expect) -{ - if (fast::util::wildcard_match(word, pattern) != expect) - { - if (expect == true) - std::cout << "ERROR: " << word << " didn't match " << pattern << std::endl; - else - std::cout << "ERROR: " << word << " matched " << pattern << std::endl; - - return false; - } - - return true; -} -} - -int main(int, char **) -{ - bool success = - Test("a", "b", false) && - Test("b", "b", true) && - Test("abc", "def", false) && - Test("def", "def", true) && - Test("def", "d?f", true) && - Test("def", "d?d", false) && - Test("def", "??d", false) && - Test("def", "d??", true) && - Test("abcdef", "a*e", false) && - Test("abcdef", "a*f", true) && - Test("abcdef", "a?c*f", true) && - Test("abcdef", "a?b*f", false) && - Test("abcdef", "a*b*f", true) && - Test("abcdef", "abc*", true) && - Test("abcdef", "*def", true); - - if (success == true) - std::cout << "wildcard_match_test: SUCCESS" << std::endl; - - return 0; -} diff --git a/fastlib/src/vespa/packages/.gitignore b/fastlib/src/vespa/packages/.gitignore deleted file mode 100644 index 1559ce1aa84..00000000000 --- a/fastlib/src/vespa/packages/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.a -*.lib -*.so.* -.depend -Makefile diff --git a/fastlib/src/vespa/packages/CMakeLists.txt b/fastlib/src/vespa/packages/CMakeLists.txt deleted file mode 100644 index f6a00c9ec06..00000000000 --- a/fastlib/src/vespa/packages/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_fast - SOURCES - INSTALL lib64 - DEPENDS - fastlib_text - fastlib_io - fastlib_util - fastlib_fast_testsuite -) diff --git a/fbench/CMakeLists.txt b/fbench/CMakeLists.txt index 77d0a5e9080..851d6706247 100644 --- a/fbench/CMakeLists.txt +++ b/fbench/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_define_module( DEPENDS fastos vespalib - staging_vespalib APPS src/fbench diff --git a/filedistribution/CMakeLists.txt b/filedistribution/CMakeLists.txt index cddeccc9574..ff6e3b28473 100644 --- a/filedistribution/CMakeLists.txt +++ b/filedistribution/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(filedistribution) +install_jar(filedistribution-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-status-filedistribution.sh vespa-status-filedistribution bin) diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java index a2c5fee1e51..748d9eb1003 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReceiver.java @@ -1,5 +1,4 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - package com.yahoo.vespa.filedistribution; import com.yahoo.config.FileReference; @@ -106,14 +105,15 @@ public class FileReceiver { } File close(long hash) { - if (hasher.getValue() != hash) { + if (hasher.getValue() != hash) throw new RuntimeException("xxhash from content (" + currentHash + ") is not equal to xxhash in request (" + hash + ")"); - } + File file = new File(fileReferenceDir, fileName); + File decompressedDir = null; try { // Unpack if necessary if (fileType == FileReferenceData.Type.compressed) { - File decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); + decompressedDir = Files.createTempDirectory(tmpDir.toPath(), "archive").toFile(); CompressedFileReference.decompress(inprogressFile, decompressedDir); moveFileToDestination(decompressedDir, fileReferenceDir); } else { @@ -130,11 +130,8 @@ public class FileReceiver { log.log(Level.SEVERE, "Failed writing file: " + e.getMessage(), e); throw new RuntimeException("Failed writing file: ", e); } finally { - try { - Files.deleteIfExists(inprogressFile.toPath()); - } catch (IOException e) { - log.log(Level.SEVERE, "Failed deleting " + inprogressFile.getAbsolutePath() + ": " + e.getMessage(), e); - } + deletePath(inprogressFile); + deletePath(decompressedDir); } return file; } @@ -196,12 +193,14 @@ public class FileReceiver { log.log(Level.SEVERE, message, e); throw new RuntimeException(message, e); } finally { - deleteFileOrDirectory(tempFile); + deletePath(tempFile); } } - private static void deleteFileOrDirectory(File path) { + private static void deletePath(File path) { + if (path == null) return; if ( ! path.exists()) return; + try { if (path.isDirectory()) IOUtils.recursiveDeleteDir(path); diff --git a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java index db6c4b70b83..03f8d184f94 100644 --- a/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java +++ b/filedistribution/src/main/java/com/yahoo/vespa/filedistribution/FileReferenceData.java @@ -5,7 +5,6 @@ import com.yahoo.config.FileReference; import java.nio.ByteBuffer; - /** * Utility class for a file reference with data and metadata * @@ -13,7 +12,7 @@ import java.nio.ByteBuffer; */ public abstract class FileReferenceData { - public enum Type {file, compressed} + public enum Type { file, compressed } private final FileReference fileReference; private final String filename; @@ -25,17 +24,11 @@ public abstract class FileReferenceData { this.type = type; } - public FileReference fileReference() { - return fileReference; - } + public FileReference fileReference() {return fileReference;} - public String filename() { - return filename; - } + public String filename() {return filename;} - public Type type() { - return type; - } + public Type type() {return type;} public ByteBuffer content() { ByteBuffer bb = ByteBuffer.allocate((int)size()); @@ -54,7 +47,7 @@ public abstract class FileReferenceData { /** * Only guaranteed to be valid after all content has been consumed. - * @return xx64hash of content + * @return xxhash64 of content */ public abstract long xxhash(); @@ -70,4 +63,8 @@ public abstract class FileReferenceData { * */ public abstract void close(); + + @Override + public String toString() { return fileReference.value() + " (" + filename + "), " + type.name(); } + } diff --git a/flags/CMakeLists.txt b/flags/CMakeLists.txt index 1b1adce2463..722dbd2679e 100644 --- a/flags/CMakeLists.txt +++ b/flags/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(flags) +install_jar(flags-jar-with-dependencies.jar) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index c7332c24a7e..3ae302f4958 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -150,6 +150,12 @@ public class Flags { "Will trigger a heap dump during if container shutdown times out", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag LOAD_CODE_AS_HUGEPAGES = defineFeatureFlag( + "load-code-as-hugepages", false, + List.of("baldersheim"), "2022-05-13", "2022-12-31", + "Will try to map the code segment with huge (2M) pages", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); public static final UnboundDoubleFlag CONTAINER_SHUTDOWN_TIMEOUT = defineDoubleFlag( "container-shutdown-timeout", 50.0, @@ -223,8 +229,8 @@ public class Flags { ZONE_ID, APPLICATION_ID); public static final UnboundStringFlag JDK_VERSION = defineStringFlag( - "jdk-version", "11", - List.of("hmusum"), "2021-10-25", "2022-05-15", + "jdk-version", "17", + List.of("hmusum"), "2021-10-25", "2022-06-01", "JDK version to use on host and inside containers. Note application-id dimension only applies for container, " + "while hostname and node type applies for host.", "Takes effect on restart for Docker container and on next host-admin tick for host", @@ -263,7 +269,7 @@ public class Flags { public static final UnboundBooleanFlag FAIL_DEPLOYMENT_WITH_INVALID_JVM_OPTIONS = defineFeatureFlag( "fail-deployment-with-invalid-jvm-options", true, - List.of("hmusum"), "2021-12-20", "2022-05-15", + List.of("hmusum"), "2021-12-20", "2022-06-01", "Whether to fail deployments with invalid JVM options in services.xml", "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index bc199f7160e..adef0110858 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -262,6 +262,12 @@ public class PermanentFlags { "Takes effect on next redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundListFlag<String> CLOUD_ACCOUNTS = defineListFlag( + "cloud-accounts", List.of(), String.class, + "A list of 12-digit AWS account IDs that are valid for the given tenant", + "Takes effect on next deployment through controller", + TENANT_ID); + private PermanentFlags() {} private static UnboundBooleanFlag defineFeatureFlag( diff --git a/functions.cmake b/functions.cmake index b8ac3497ff2..3721d26d0df 100644 --- a/functions.cmake +++ b/functions.cmake @@ -655,18 +655,14 @@ function(install_config_definitions) install(DIRECTORY ${DEFINITIONS_DIR}/ DESTINATION share/vespa/configdefinitions FILES_MATCHING PATTERN "*.def") endfunction() -function(install_java_artifact NAME) - install(FILES "target/${NAME}.jar" DESTINATION lib/jars/) +function(install_jar FILENAME) + install(FILES "target/${FILENAME}" DESTINATION lib/jars/) endfunction() -function(install_java_artifact_dependencies NAME) +function(install_jar_dependencies NAME) install(DIRECTORY "target/dependency/" DESTINATION lib/jars FILES_MATCHING PATTERN "*.jar") endfunction() -function(install_fat_java_artifact NAME) - install(FILES "target/${NAME}-jar-with-dependencies.jar" DESTINATION lib/jars/) -endfunction() - function(install_absolute_symlink TARGET LINK) install(CODE "execute_process(COMMAND ln -snf ${TARGET} \$ENV{DESTDIR}${CMAKE_INSTALL_PREFIX}/${LINK})") endfunction(install_absolute_symlink) @@ -686,9 +682,23 @@ function(install_configserver_component NAME) if(NOT PARAM_CLASSIFIER) SET(PARAM_CLASSIFIER "jar-with-dependencies") endif() - install(FILES "target/${NAME}-${PARAM_CLASSIFIER}.jar" DESTINATION lib/jars/) - install(DIRECTORY DESTINATION conf/configserver-app/components) - install_symlink(lib/jars/${NAME}-${PARAM_CLASSIFIER}.jar conf/configserver-app/components/${NAME}.jar) + install_jar("${NAME}-${PARAM_CLASSIFIER}.jar") + install_app_component_symlink(configserver-app "${NAME}" CLASSIFIER "${PARAM_CLASSIFIER}") +endfunction() + +function(install_app_component_symlink APPNAME NAME) + cmake_parse_arguments( + PARAM + "" + "CLASSIFIER" + "" + ${ARGN} + ) + if(NOT PARAM_CLASSIFIER) + SET(PARAM_CLASSIFIER "jar-with-dependencies") + endif() + install(DIRECTORY DESTINATION conf/${APPNAME}/components) + install_symlink(lib/jars/${NAME}-${PARAM_CLASSIFIER}.jar conf/${APPNAME}/components/${NAME}.jar) endfunction() function(add_extra_projects) diff --git a/hosted-tenant-base/pom.xml b/hosted-tenant-base/pom.xml index 3c17bb840c1..ad61e1c28c0 100644 --- a/hosted-tenant-base/pom.xml +++ b/hosted-tenant-base/pom.xml @@ -235,12 +235,11 @@ <!-- note: ordering affects how overrides are evaluated! --> <testBundleScopeOverrides> ${extraTestBundleScopeOverrides}, - com.yahoo.vespa:application:test, - com.yahoo.vespa:container-test:runtime, org.junit.jupiter:junit-jupiter-api:provided, - org.opentest4j:opentest4j:test, org.junit.jupiter:junit-jupiter-engine:test, org.junit.vintage:junit-vintage-engine:test, + com.yahoo.vespa:application:test, + com.yahoo.vespa:container-test:runtime, com.yahoo.vespa:vespa-feed-client:runtime, <!-- prevent effective compile scope of vespa-feed-client in test bundle --> com.yahoo.vespa:vespa-feed-client-api:provided, com.yahoo.vespa:tenant-cd-api:provided diff --git a/hosted-zone-api/CMakeLists.txt b/hosted-zone-api/CMakeLists.txt index ebbe7d806a1..914101b970f 100644 --- a/hosted-zone-api/CMakeLists.txt +++ b/hosted-zone-api/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(hosted-zone-api) +install_jar(hosted-zone-api-jar-with-dependencies.jar) diff --git a/http-client/CMakeLists.txt b/http-client/CMakeLists.txt index 3f5bf469481..58249a8c665 100644 --- a/http-client/CMakeLists.txt +++ b/http-client/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(http-client) +install_jar(http-client-jar-with-dependencies.jar) diff --git a/integration/intellij/build.gradle b/integration/intellij/build.gradle index 1c2dae46d87..4374f6f964a 100644 --- a/integration/intellij/build.gradle +++ b/integration/intellij/build.gradle @@ -36,7 +36,7 @@ compileJava { } group 'ai.vespa' -version '1.1.5' // Also update pom.xml version if this is changed +version '1.1.6' // Also update pom.xml version if this is changed sourceCompatibility = 11 @@ -64,8 +64,10 @@ patchPluginXml { untilBuild = '213.*' // in changeNotes you can add a description of the changes in this version (would appear in the plugin page in preferences\plugins) changeNotes = """ - Support for inputs in rank profiles. - Correct parsing of lambda expressions. + Support for default values in inputs. + Support for unified constant syntax. + Support all tensor formats. + Support tensor generate functions. """ } diff --git a/integration/intellij/pom.xml b/integration/intellij/pom.xml index 1c973b84d37..3f2a4c24771 100644 --- a/integration/intellij/pom.xml +++ b/integration/intellij/pom.xml @@ -9,7 +9,7 @@ <relativePath>../parent/pom.xml</relativePath> </parent> <artifactId>vespa-intellij</artifactId> <!-- Not used - plugin is build by gradle --> - <version>1.1.5</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle --> + <version>1.1.6</version> <!-- See copy-zip below, which depends on this being the same as the v. in build.gradle --> <description> Maven wrapper for the gradle build of this IntelliJ plugin. </description> diff --git a/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf index b0e0e5e61dc..87c4a012fb5 100644 --- a/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf +++ b/integration/intellij/src/main/bnf/ai/vespa/intellij/schema/parser/sd.bnf @@ -91,7 +91,7 @@ private UriPath ::= ('H'|'h') ('T'|'t') ('T'|'t') ('P'|'p') ('S'|'s')? ':' ('//' OnnxModelDefinition ::= onnx-model IdentifierVal '{' OnnxModelBody '}' OnnxModelBody ::= OnnxModelBodyOptions* private OnnxModelBodyOptions ::= (file ':' FilePath) | (uri ':' UriPath) | - ((input | output) (IdentifierVal | STRING_REG) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG)) + ((input | output) (RankFeature | IdentifierVal | STRING_REG) ':' ('.' | '/' | '(' | ')' | IdentifierWithDashVal | WORD_REG)) SchemaAnnotationDefinition ::= AnnotationDefinition { mixin="ai.vespa.intellij.schema.psi.impl.SdNamedElementImpl" @@ -110,7 +110,7 @@ AnnotationFieldDefinition ::= field IdentifierVal type FieldTypeName '{' '}' // NOTE: These must end by "Expr" - see this line above: extends(".*Expr")=RankingExpression // The *Expr alternatives are consumed greedily so order matters. //------------------------- -RankingExpression ::= LiteralTensorExpr | FilePathExpr | ParenthesisedExpr | BooleanExpr | ArithmeticExpr | IfFunctionExpr | +RankingExpression ::= LiteralOrGenerateTensorExpr | FilePathExpr | ParenthesisedExpr | BooleanExpr | ArithmeticExpr | IfFunctionExpr | QueryDefinitionExpr | FunctionCallOrLambdaExpr | InListRankingExpr | PrimitiveExpr | SliceExpr FilePathExpr ::= file ':' (FilePath | WordWrapper) @@ -139,6 +139,8 @@ PrimitiveExpr ::= ( (('-')? INTEGER_REG) | (('-')? FLOAT_REG) | IdentifierVal | SliceExpr ::= RankingExpression ( SliceKey | SliceIndex | FullTensorAddress ) +GenerateExpr ::= TensorType ParenthesisedExpr + SliceKey ::= '{' Label '}' SliceIndex ::= '[' RankingExpression ']' @@ -148,13 +150,16 @@ KeyValue ::= IdentifierVal ':' ( Label | RankingExpression ) Label ::= IdentifierVal | STRING_REG -LiteralTensorExpr ::= TensorType ':' '{' TensorValue ( ',' TensorValue )* '}' - -TensorValue ::= CellAddress ':' RankingExpression - -CellAddress ::= Label | FullTensorAddress +LiteralOrGenerateTensorExpr ::= TensorType ( + ( ':' TensorValue ) | // literal verbose form tensor + ParenthesisedExpr // generate tensor + ) -LambdaExpr ::= IdentifierWithDashVal '(' IdentifierVal (COMMA IdentifierVal)* ')' ParenthesisedExpr +TensorValue ::= MappedTensorValue | ArrayTensorValues +MappedTensorValue ::= '{' MappedTensorBlock ( ',' MappedTensorBlock )* '}' +MappedTensorBlock ::= TensorAddress ':' ( FLOAT_REG | ArrayTensorValues ) +ArrayTensorValues ::= '[' ( FLOAT_REG | ArrayTensorValues ) ( ',' ( FLOAT_REG | ArrayTensorValues ) )* ']' +TensorAddress ::= Label | FullTensorAddress //------------------------- //-- Rank Profile rules --- @@ -187,8 +192,10 @@ private MinHitsDefinition ::= min-hits-per-thread ':' ('-')? INTEGER_REG private NumSearchPartitionDefinition ::= num-search-partition ':' INTEGER_REG FieldWeightDefinition ::= weight DottedIdentifiers ':' INTEGER_REG StrictDefinition ::= strict ':' (true | false) -InputsDefinition ::= inputs '{' InputsBody '}' -InputsBody ::= (QueryDefinition TensorType)* +InputsDefinition ::= inputs '{' InputDefinition* '}' +InputDefinition ::= ( QueryDefinition | IdentifierVal) + (':')? + ( TensorType | "double" )? (':' ( TensorValue | FLOAT_REG | INTEGER_REG) )? FirstPhaseDefinition ::= first-phase '{' FirstPhaseBody '}' { mixin="ai.vespa.intellij.schema.psi.impl.SdFirstPhaseDefinitionMixin" } FirstPhaseBody ::= FirstPhaseBodyOptions* // Does not support zero-or-one occurrences @@ -216,12 +223,16 @@ ArgumentDefinition ::= IdentifierVal } SummaryFeaturesDefinition ::= summary-features ((':' RankFeature+) | ((inherits IdentifierWithDashVal)? '{' RankFeature* '}')) - +i MatchFeaturesDefinition ::= match-features ((':' RankFeature+) | ((inherits IdentifierWithDashVal)? '{' RankFeature* '}')) RankFeaturesDefinition ::= rank-features ((':' RankFeature+) | ('{' RankFeature* '}')) -ConstantsDefinition ::= constants '{' (IdentifierVal ':' RankPropertiesValue)* '}' +ConstantsDefinition ::= constants '{' InnerConstantDefinition* '}' + +InnerConstantDefinition ::= ( ("constant" '(' IdentifierVal ')') | IdentifierVal ) + (':')? ( TensorType | "double" )? + (':')? ( TensorValue | FLOAT_REG | INTEGER_REG | (file ':' FilePath) | (uri ':' UriPath)) RankFeature ::= QueryDefinition | ItemRawScoreDefinition | FunctionCallExpr | DottedIdentifierWithDash QueryDefinition ::= "query" '(' IdentifierWithDashVal ')' diff --git a/jdisc-cloud-aws/CMakeLists.txt b/jdisc-cloud-aws/CMakeLists.txt index d0c55306d12..d83f371c014 100644 --- a/jdisc-cloud-aws/CMakeLists.txt +++ b/jdisc-cloud-aws/CMakeLists.txt @@ -1,3 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(jdisc-cloud-aws) +install_jar(jdisc-cloud-aws-jar-with-dependencies.jar) install_config_definitions() diff --git a/jdisc-security-filters/CMakeLists.txt b/jdisc-security-filters/CMakeLists.txt index d37c7b18cf8..3927d07613a 100644 --- a/jdisc-security-filters/CMakeLists.txt +++ b/jdisc-security-filters/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(jdisc-security-filters) +install_jar(jdisc-security-filters-jar-with-dependencies.jar) install_config_definitions() diff --git a/jdisc_core/CMakeLists.txt b/jdisc_core/CMakeLists.txt index 073649a5e60..b70c5744e61 100644 --- a/jdisc_core/CMakeLists.txt +++ b/jdisc_core/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(jdisc_core) +install_jar(jdisc_core-jar-with-dependencies.jar) diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java index bc243afef38..af57cf39e73 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/HeaderFields.java @@ -1,8 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jdisc; -import com.google.common.collect.ImmutableList; - import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -264,7 +262,7 @@ public class HeaderFields implements Map<String, List<String>> { list.add(new MyEntry(key, value)); } } - return ImmutableList.copyOf(list); + return List.copyOf(list); } @Override diff --git a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java index dda33c2d1a8..3a706101790 100644 --- a/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java +++ b/jdisc_core/src/main/java/com/yahoo/jdisc/core/ExportPackages.java @@ -58,7 +58,9 @@ public class ExportPackages { .append("org.aopalliance.aop"); for (int i = 1; i < jars.length; ++i) { - out.append(", ").append(getExportedPackages(jars[i])); + String exports = getExportedPackages(jars[i]); + if (exports != null && ! exports.isEmpty()) + out.append(", ").append(exports); } return out.toString(); } diff --git a/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/LogFrameworksIntegrationTest.java b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/LogFrameworksIntegrationTest.java new file mode 100644 index 00000000000..6297a9c4505 --- /dev/null +++ b/jdisc_core_test/integration_test/src/test/java/com/yahoo/jdisc/core/LogFrameworksIntegrationTest.java @@ -0,0 +1,56 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.jdisc.core; + +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.test.TestDriver; +import com.yahoo.log.LogSetup; +import org.junit.Assert; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.util.List; + +import static org.junit.Assert.assertTrue; + + +/** + * @author Simon Thoresen Hult + * @author gjoranv + */ +public class LogFrameworksIntegrationTest { + private static final String VESPA_LOG_TARGET = "vespa.log.target"; + + @Rule + public TemporaryFolder tempFolder = new TemporaryFolder(); + + @Test + public void requireThatAllSupportedLogFrameworksAreConfigured() throws Exception { + File logFile = tempFolder.newFile(); + try { + System.setProperty(VESPA_LOG_TARGET, "file:" + logFile.getAbsolutePath()); + TestDriver driver = TestDriver.newApplicationBundleInstance("app-h-log.jar", false); + + List<String> logLines = IOUtils.getLines(logFile.getAbsolutePath()); + assertLogFileContains("[jdk14] hello world", logLines); + assertLogFileContains("[slf4j] hello world", logLines); + assertLogFileContains("[log4j] hello world", logLines); + assertLogFileContains("[jcl] hello world", logLines); + + assertTrue(driver.close()); + } finally { + System.clearProperty(VESPA_LOG_TARGET); + // Triggers a warning that can be ignored. Necessary to reset the log target for later tests. + LogSetup.initVespaLogging("jdisc_core_test"); + } + } + + private static void assertLogFileContains(String expected, List<String> logLines) { + for (var line : logLines) { + if (line.contains(expected)) return; + } + Assert.fail("Log did not contain : " + expected); + } + +} diff --git a/jdisc_jetty/CMakeLists.txt b/jdisc_jetty/CMakeLists.txt index 129cd9de06a..7f72c61be20 100644 --- a/jdisc_jetty/CMakeLists.txt +++ b/jdisc_jetty/CMakeLists.txt @@ -1,3 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact(jdisc_jetty) -install_java_artifact_dependencies(jdisc_jetty) +install_jar(jdisc_jetty.jar) +install_jar_dependencies(jdisc_jetty) diff --git a/jdisc_jetty/pom.xml b/jdisc_jetty/pom.xml index ea4aa82cb15..76140f67416 100644 --- a/jdisc_jetty/pom.xml +++ b/jdisc_jetty/pom.xml @@ -29,6 +29,10 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-continuation</artifactId> </dependency> <dependency> diff --git a/jrt/src/com/yahoo/jrt/Supervisor.java b/jrt/src/com/yahoo/jrt/Supervisor.java index 48dd2d8bd3d..65deea0dc61 100644 --- a/jrt/src/com/yahoo/jrt/Supervisor.java +++ b/jrt/src/com/yahoo/jrt/Supervisor.java @@ -12,7 +12,7 @@ import java.util.concurrent.atomic.AtomicReference; * requests obtained from that {@link Target}. Note that RPC * invocations can be performed both ways across a connection, so even * the client side of a connection has RPC server capabilities. - **/ + */ public class Supervisor { private final Transport transport; @@ -23,23 +23,22 @@ public class Supervisor { private boolean dropEmptyBuffers = false; /** - * Create a new Supervisor based on the given {@link Transport} + * Creates a new Supervisor based on the given {@link Transport} * - * @param transport object performing low-level operations for - * this Supervisor - **/ + * @param transport object performing low-level operations for this Supervisor + */ public Supervisor(Transport transport) { this.transport = transport; new MandatoryMethods(this); } /** - * Drop empty buffers. This will reduce memory footprint for idle + * Drops empty buffers. This will reduce memory footprint for idle * connections at the cost of extra allocations when buffer space * is needed again. * * @param value true means drop empty buffers - **/ + */ public Supervisor setDropEmptyBuffers(boolean value) { dropEmptyBuffers = value; return this; @@ -47,7 +46,7 @@ public class Supervisor { boolean getDropEmptyBuffers() { return dropEmptyBuffers; } /** - * Set maximum input buffer size. This value will only affect + * Sets maximum input buffer size. This value will only affect * connections that use a common input buffer when decoding * incoming packets. Note that this value is not an absolute * max. The buffer will still grow larger than this value if @@ -55,14 +54,14 @@ public class Supervisor { * larger than this value, it will be shrunk back when possible. * * @param bytes buffer size in bytes. 0 means unlimited. - **/ + */ public void setMaxInputBufferSize(int bytes) { maxInputBufferSize = bytes; } int getMaxInputBufferSize() { return maxInputBufferSize; } /** - * Set maximum output buffer size. This value will only affect + * Sets maximum output buffer size. This value will only affect * connections that use a common output buffer when encoding * outgoing packets. Note that this value is not an absolute * max. The buffer will still grow larger than this value if needed @@ -70,35 +69,35 @@ public class Supervisor { * than this value, it will be shrunk back when possible. * * @param bytes buffer size in bytes. 0 means unlimited. - **/ + */ public void setMaxOutputBufferSize(int bytes) { maxOutputBufferSize = bytes; } int getMaxOutputBufferSize() { return maxOutputBufferSize; } /** - * Obtain the method map for this Supervisor + * Obtains the method map for this Supervisor * * @return the method map - **/ + */ HashMap<String, Method> methodMap() { return methodMap.getAcquire(); } /** - * Obtain the underlying Transport object. + * Obtains the underlying Transport object. * * @return underlying Transport object - **/ + */ public Transport transport() { return transport; } /** - * Add a method to the set of methods held by this Supervisor + * Adds a method to the set of methods held by this Supervisor * * @param method the method to add - **/ + */ public void addMethod(Method method) { synchronized (methodMapLock) { HashMap<String, Method> newMap = new HashMap<>(methodMap()); @@ -108,12 +107,12 @@ public class Supervisor { } /** - * Remove a method from the set of methods held by this + * Removes a method from the set of methods held by this * Supervisor. Use this if you know exactly which method to remove * and not only the name. * * @param method the method to remove - **/ + */ public void removeMethod(Method method) { synchronized (methodMapLock) { HashMap<String, Method> newMap = new HashMap<>(methodMap()); @@ -124,19 +123,19 @@ public class Supervisor { } /** - * Connect to the given address. The new {@link Target} will be + * Connects to the given address. The new {@link Target} will be * associated with this Supervisor. * * @return Target representing our end of the connection * @param spec where to connect * @see #connect(com.yahoo.jrt.Spec, java.lang.Object) - **/ + */ public Target connect(Spec spec) { return transport.connect(this, spec, null); } /** - * Connect to the given address. The new {@link Target} will be + * Connects to the given address. The new {@link Target} will be * associated with this Supervisor and will have 'context' as * application context. * @@ -144,18 +143,18 @@ public class Supervisor { * @param spec where to connect * @param context application context for the Target * @see Target#getContext - **/ + */ public Target connect(Spec spec, Object context) { return transport.connect(this, spec, context); } /** - * Listen to the given address. + * Listens to the given address. * * @return active object accepting new connections that will be * associated with this Supervisor * @param spec the address to listen to - **/ + */ public Acceptor listen(Spec spec) throws ListenFailedException { return transport.listen(this, spec); } @@ -165,7 +164,7 @@ public class Supervisor { * is empty and only used for testing through sub-classing. * * @param info information about the written packet - **/ + */ void writePacket(PacketInfo info) {} /** @@ -173,17 +172,17 @@ public class Supervisor { * is empty and only used for testing through sub-classing. * * @param info information about the read packet - **/ + */ void readPacket(PacketInfo info) {} /** - * Handle a packet received on one of the connections associated + * Handles a packet received on one of the connections associated * with this Supervisor. This method is invoked for all packets * not handled by a {@link ReplyHandler} * * @param conn where the packet came from * @param packet the packet - **/ + */ void handlePacket(Connection conn, Packet packet) { if (packet.packetCode() != Packet.PCODE_REQUEST) { return; @@ -195,4 +194,5 @@ public class Supervisor { packet.requestId(), packet.noReply()).invoke(); } + } diff --git a/jrt/src/com/yahoo/jrt/Target.java b/jrt/src/com/yahoo/jrt/Target.java index 1bc62d12ac9..a59aa341fe0 100644 --- a/jrt/src/com/yahoo/jrt/Target.java +++ b/jrt/src/com/yahoo/jrt/Target.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.util.Optional; /** @@ -10,54 +9,54 @@ import java.util.Optional; * side. The client side is the one initiating the connection. RPC * requests may be invoked across the connection from both the client * and the server side. - **/ + */ public abstract class Target { private Object context; /** - * Create a Target with the given application context. + * Creates a Target with the given application context. * * @param context application context - **/ + */ Target(Object context) { this.context = context; } /** - * Create a Target without any application context. - **/ + * Creates a Target without any application context. + */ Target() { this(null); } /** - * Set the application context associated with this target. + * Sets the application context associated with this target. * - * @param context application context - **/ + * @param context the application context + */ public void setContext(Object context) { this.context = context; } /** - * Obtain the application context associated with this target. + * Obtains the application context associated with this target. * - * @return application context - **/ + * @return the application context + */ public Object getContext() { return context; } /** - * Check if this target is still valid for invocations. + * Checks if this target is still valid for invocations. * * @return true if this target is still valid - **/ + */ public abstract boolean isValid(); /** - * Obtain the low-level reason behind losing the connection for + * Obtains the low-level reason behind losing the connection for * which this target is an endpoint. If the target is still valid * or if the target became invalid because it was closed, this * method will return null. In other cases this method may or may @@ -66,11 +65,11 @@ public abstract class Target { * based on implementation details across platforms. * * @return exception causing connection loss or null - **/ + */ public Exception getConnectionLostReason() { return null; } /** - * @return the security context associated with this target, or empty if no connection or is insecure. + * Returns the security context associated with this target, or empty if no connection or is insecure. */ public abstract Optional<SecurityContext> getSecurityContext(); @@ -79,7 +78,7 @@ public abstract class Target { * connection. * * @return true if this is a client-side target - **/ + */ public abstract boolean isClient(); /** @@ -87,7 +86,7 @@ public abstract class Target { * connection. * * @return true if this is a server-side target - **/ + */ public abstract boolean isServer(); /** @@ -95,7 +94,7 @@ public abstract class Target { * * @param req the request * @param timeout timeout in seconds - **/ + */ public abstract void invokeSync(Request req, double timeout); /** @@ -105,7 +104,7 @@ public abstract class Target { * @param req the request * @param timeout timeout in seconds * @param waiter callback handler - **/ + */ public abstract void invokeAsync(Request req, double timeout, RequestWaiter waiter); @@ -115,10 +114,10 @@ public abstract class Target { * ignored. However, the return value gives a little hint by * indicating whether the invocation has been attempted at all. * + * @param req the request * @return false if the invocation was not attempted due to the * target being invalid - * @param req the request - **/ + */ public abstract boolean invokeVoid(Request req); /** @@ -128,9 +127,9 @@ public abstract class Target { * returned. Multiple adds of the same watcher has no additional * effect. * - * @return true if the add operation was performed * @param watcher the watcher to be added - **/ + * @return true if the add operation was performed + */ public abstract boolean addWatcher(TargetWatcher watcher); /** @@ -139,16 +138,17 @@ public abstract class Target { * and false is returned. Multiple removes of the same watcher has * no additional effect. * - * @return true if the remove operation was performed * @param watcher the watcher to be removed + * @return true if the remove operation was performed * @see #addWatcher - **/ + */ public abstract boolean removeWatcher(TargetWatcher watcher); /** * Close this target. Note that the close operation is * asynchronous. If you need to wait for the target to become * invalid, use the {@link Transport#sync Transport.sync} method. - **/ + */ public abstract void close(); + } diff --git a/jrt/src/com/yahoo/jrt/Transport.java b/jrt/src/com/yahoo/jrt/Transport.java index 0c8977f2c40..871d0188691 100644 --- a/jrt/src/com/yahoo/jrt/Transport.java +++ b/jrt/src/com/yahoo/jrt/Transport.java @@ -4,6 +4,7 @@ package com.yahoo.jrt; import java.nio.channels.SocketChannel; import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Level; @@ -15,7 +16,7 @@ import java.util.logging.Logger; * multiplexed network IO, handles scheduled tasks and keeps track of * some additional helper threads. A single Transport object can back * multiple {@link Supervisor} objects. - **/ + */ public class Transport { private static final Logger log = Logger.getLogger(Transport.class.getName()); @@ -30,11 +31,11 @@ public class Transport { private final int eventsBeforeWakeup; private final TransportMetrics metrics = TransportMetrics.getInstance(); - private final ArrayList<TransportThread> threads = new ArrayList<>(); + private final List<TransportThread> threads = new ArrayList<>(); private final Random rnd = new Random(); /** - * Create a new Transport object with the given fatal error + * Creates a new Transport object with the given fatal error * handler and CryptoEngine. If a fatal error occurs when no fatal * error handler is registered, the default action is to log the * error and exit with exit code 1. @@ -44,7 +45,7 @@ public class Transport { * @param cryptoEngine crypto engine to use * @param numThreads number of {@link TransportThread}s. * @param eventsBeforeWakeup number write events in Q before waking thread up - **/ + */ public Transport(String name, FatalErrorHandler fatalHandler, CryptoEngine cryptoEngine, int numThreads, boolean tcpNoDelay, int eventsBeforeWakeup) { this.name = name; this.fatalHandler = fatalHandler; // NB: this must be set first @@ -82,7 +83,7 @@ public class Transport { * Select a random transport thread * * @return a random transport thread - **/ + */ public TransportThread selectThread() { return threads.get(rnd.nextInt(threads.size())); } @@ -93,24 +94,24 @@ public class Transport { public String getName() { return name; } /** - * Use the underlying CryptoEngine to create a CryptoSocket for + * Uses the underlying CryptoEngine to create a CryptoSocket for * the client side of a connection. * * @return CryptoSocket handling appropriate encryption * @param channel low-level socket channel to be wrapped by the CryptoSocket * @param spec who we are connecting to, for hostname validation - **/ + */ CryptoSocket createClientCryptoSocket(SocketChannel channel, Spec spec) { return cryptoEngine.createClientCryptoSocket(channel, spec); } /** - * Use the underlying CryptoEngine to create a CryptoSocket for + * Uses the underlying CryptoEngine to create a CryptoSocket for * the server side of a connection. * * @return CryptoSocket handling appropriate encryption * @param channel low-level socket channel to be wrapped by the CryptoSocket - **/ + */ CryptoSocket createServerCryptoSocket(SocketChannel channel) { return cryptoEngine.createServerCryptoSocket(channel); } @@ -122,7 +123,7 @@ public class Transport { * * @param problem the throwable causing the failure * @param context the object owning the crashing thread - **/ + */ void handleFailure(Throwable problem, Object context) { if (fatalHandler != null) { fatalHandler.handleFailure(problem, context); @@ -135,19 +136,19 @@ public class Transport { } /** - * Listen to the given address. This method is called by a {@link + * Listens to the given address. This method is called by a {@link * Supervisor} object. * * @return active object accepting new connections * @param owner the one calling this method * @param spec the address to listen to - **/ + */ Acceptor listen(Supervisor owner, Spec spec) throws ListenFailedException { return new Acceptor(this, owner, spec); } /** - * Connect to the given address. This method is called by a {@link + * Connects to the given address. This method is called by a {@link * Supervisor} object. * * @return the new connection @@ -166,7 +167,7 @@ public class Transport { } /** - * Request that {@link Connection#doHandshakeWork()} be called (in any thread) + * Requests that {@link Connection#doHandshakeWork()} be called (in any thread) * followed by a call to {@link Connection#handleHandshakeWorkDone()} from the transport thread. * * @param conn the connection needing handshake work @@ -176,7 +177,7 @@ public class Transport { } /** - * Synchronize with all transport threads. This method will block + * Synchronizes with all transport threads. This method will block * until all commands issued before this method was invoked has * completed. If a transport thread has been shut down (or is in * the progress of being shut down) this method will instead wait @@ -185,7 +186,7 @@ public class Transport { * method from a transport thread is not a good idea. * * @return this object, to enable chaining - **/ + */ public Transport sync() { for (TransportThread thread: threads) { thread.sync(); @@ -194,10 +195,10 @@ public class Transport { } /** - * Initiate controlled shutdown of all transport threads. + * Initiates controlled shutdown of all transport threads. * * @return this object, to enable chaining with join - **/ + */ public Transport shutdown() { connector.close(); for (TransportThread thread: threads) { @@ -207,8 +208,8 @@ public class Transport { } /** - * Wait for all transport threads to finish. - **/ + * Waits for all transport threads to finish. + */ public void join() { for (TransportThread thread: threads) { thread.join(); @@ -225,4 +226,5 @@ public class Transport { public TransportMetrics metrics() { return metrics; } + } diff --git a/jrt/src/com/yahoo/jrt/TransportThread.java b/jrt/src/com/yahoo/jrt/TransportThread.java index a3f1773c814..6063e72ecdd 100644 --- a/jrt/src/com/yahoo/jrt/TransportThread.java +++ b/jrt/src/com/yahoo/jrt/TransportThread.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.jrt; - import java.io.IOException; import java.nio.channels.SelectionKey; import java.nio.channels.Selector; @@ -9,11 +8,10 @@ import java.util.Iterator; import java.util.logging.Level; import java.util.logging.Logger; - /** * A single reactor/scheduler thread inside a potentially * multi-threaded {@link Transport}. - **/ + */ public class TransportThread { private static final int OPEN = 1; @@ -192,17 +190,17 @@ public class TransportThread { * * @param problem the throwable causing the failure * @param context the object owning the crashing thread - **/ + */ void handleFailure(Throwable problem, Object context) { parent.handleFailure(problem, context); } /** - * Add a connection to the set of connections handled by this + * Adds a connection to the set of connections handled by this * TransportThread. Invoked by the {@link Connector} class. * * @param conn the connection to add - **/ + */ void addConnection(Connection conn) { if (!postCommand(new AddConnectionCmd(conn))) { perform(new CloseConnectionCmd(conn)); @@ -210,20 +208,20 @@ public class TransportThread { } /** - * Request an asynchronous close of a connection. + * Requests an asynchronous close of a connection. * * @param conn the connection to close - **/ + */ void closeConnection(Connection conn) { postCommand(new CloseConnectionCmd(conn)); } /** - * Request an asynchronous enabling of write events for a + * Requests an asynchronous enabling of write events for a * connection. * * @param conn the connection to enable write events for - **/ + */ void enableWrite(Connection conn) { if (Thread.currentThread() == thread) { handleEnableWrite(conn); @@ -237,24 +235,24 @@ public class TransportThread { } /** - * Create a {@link Task} that can be scheduled for execution in + * Creates a {@link Task} that can be scheduled for execution in * the transport thread. * * @return the newly created Task * @param cmd what to run when the task is executed - **/ + */ public Task createTask(Runnable cmd) { return new Task(scheduler, cmd); } /** - * Perform the given command in such a way that it does not run + * Performs the given command in such a way that it does not run * concurrently with the transport thread or other commands * performed by invoking this method. This method will continue to * work even after the transport thread has been shut down. * * @param cmd the command to perform - **/ + */ public void perform(Runnable cmd) { if (Thread.currentThread() == thread) { cmd.run(); @@ -269,16 +267,16 @@ public class TransportThread { } /** - * Wake up this transport thread explicitly. - **/ + * Wakes up this transport thread explicitly. + */ public void wakeup() { selector.wakeup(); } /** - * Wake up this transport thread explicitly, but only if the + * Wakes up this transport thread explicitly, but only if the * calling thread is not the transport thread itself. - **/ + */ public void wakeup_if_not_self() { if (Thread.currentThread() != thread) { wakeup(); @@ -286,7 +284,7 @@ public class TransportThread { } /** - * Synchronize with the transport thread. This method will block + * Synchronizes with the transport thread. This method will block * until all commands issued before this method was invoked has * completed. If the transport thread has been shut down (or is in * the progress of being shut down) this method will instead wait @@ -295,7 +293,7 @@ public class TransportThread { * method from the transport thread is not a good idea. * * @return this object, to enable chaining - **/ + */ public TransportThread sync() { SyncCmd cmd = new SyncCmd(); if (postCommand(cmd)) { @@ -366,4 +364,5 @@ public class TransportThread { } catch (InterruptedException e) {} } } + } diff --git a/juniper/.gitignore b/juniper/.gitignore deleted file mode 100644 index a106509440d..00000000000 --- a/juniper/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*.dsp -Makefile.ini -config_command.sh -include -lib -project.dsw -Makefile diff --git a/juniper/CMakeLists.txt b/juniper/CMakeLists.txt deleted file mode 100644 index 34dea870ec8..00000000000 --- a/juniper/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - fastlib_fast - vespalog - - LIBS - src/vespa/juniper - - TESTS - src/test -) diff --git a/juniper/INSTALL.txt b/juniper/INSTALL.txt deleted file mode 100644 index 284b3eb38b2..00000000000 --- a/juniper/INSTALL.txt +++ /dev/null @@ -1,49 +0,0 @@ -Juniper configuration and build instructions --------------------------------------------------- - -Juniper basically adheres to the standard FastOS configure/build environment. - -configure --help - -gives a list of configure options. - -Juniper supports most of the common build targets. - -Default configure parameters are set to assume that other dependent Fast modules -are located at the same directory level as this modul. -Default is to expect the minimal required set of modules. - -Prerequisites: ----------------- -Common module dependencies: - -fastos and fastlib must have been built and installed. - -This version of Juniper requires fastlib 1.5.16 or higher. -To be able to utilize the functionality for building in a different -directory, fastos <= 1.6.0 is required. - -Postgres Document Store (optional,obsolete) ---------------------------------------------- -To build with Document Store support, Postgres must have been installed and -path to this installation provided with the --pgsql-dir option. Default is -to view document store as external to this module. - - -libiconv (optional) --------------------------- -The 3rdparty module libiconv is needed to run the unit tests in the src/tests directory. - - -To build/install this module: ------------------------------------ - -1) run configure - - A shell script named config_command.sh / config_command.bat will be updated to contain last - used configure command to avoid having to re-remember the configure - options currently in use. - -2) make depend - -3) make install diff --git a/juniper/OWNERS b/juniper/OWNERS deleted file mode 100644 index 6aeaba5f2a8..00000000000 --- a/juniper/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -geirst -baldersheim diff --git a/juniper/README b/juniper/README deleted file mode 100644 index f044cad83e8..00000000000 --- a/juniper/README +++ /dev/null @@ -1,36 +0,0 @@ - -This module implements a result processing solution to create -dynamic teasers and proximity boosting of documents based on the -input query. - -The module offers the following main interface: - -rpinterface.h - - This interface provides access for Juniper to full query structure information. - The query provider needs to implement the IQuery interface to allow Juniper - to traverse the query and possibly obtain additional information about each term. - (see query.h which is included by rpinterface.h) - - This is the preferred result processing interface to Juniper. - - src/rpclient/testclient.cpp - - is an example of use of this interface. - - -Directory structure: -------------------------- -build - contains the fbuild official build specification for Juniper -doc - contains manually written and generated (doxygen) documentation - for Juniper. -include - default public include file location -lib - default library location -src - the main source directory -src/rpclient - example and regression test client -src/test - unit and submodule tests using the Fastlib test framework -src/util - contains some debug utilities for debugging/converting between - unicode/iso character sets. - -See INSTALL.txt for build instructions. - diff --git a/juniper/doc/written/fsearchparams.html b/juniper/doc/written/fsearchparams.html deleted file mode 100644 index 1a968e55c41..00000000000 --- a/juniper/doc/written/fsearchparams.html +++ /dev/null @@ -1,488 +0,0 @@ -<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> -<title>Juniper Configuration Documentation</title> -<h1>Juniper Configuration Documentation</h1> - -<b>Note:</b> This document describes in details the functionality of Juniper v.2.1.0. -The document has gradually become more and more for internal use for -instance for detailed tuning -by Professional Service. A more high level and less detailed user level -configuration documentation is also available. -<p> -Juniper implements a combined proximity ranking and dynamic teaser result -processing module.This module is intended to be interfaced to by different -Fast software modules on demand. Currently, the only available module that -makes use of Juniper is the Fast Server module, in which Juniper currently -is an integrated part of <dfn>fsearch</dfn> (the search engine executable -that runs on each search node in the system). - -<h2>Juniper simple description of functionality/implementation</h2> - -The document body is stripped for markup during document processing and -stored as an extended document summary field. -A max limit of how much of the document that gets stored is configurable as -of Fast Server v.4.17 (see the Fast Server�configuration documentation for -details). For each document -on the result page, this document extract is retrieved and fed through -Juniper which will perform the following steps: - -<ol> - <li> Scan the stripped document text (docsum) for matches of the query, - create a data structure containing information about those matches, - and provide a quality measure (rank boost value) that can be used as a - metric to determine the quality of the document wrt. proximity and - position of the search string in the document. The data structure - contains ao. a list of matches of the query ordered by quality (see below - for the quality measure). The document quality measure is computed from the - quality measure of the best of the individual matches and the total - number of hits within the document. - - <li> Generate a dynamic teaser based on the data structure previously - generated. The dynamic teaser is composed of a number of text segments - that include the "best" matches of the query in that - document. The teaser is presented with the query words highlighted. - The definition of highlight is configurable. If the document is short - enough to fit completely into the configured teaser length, it will be - provided as is, but with highlighting of the relevant keywords. -</ol> - -Step 2 is only necessary if the teaser is going to be displayed, which -might be a decision taken on basis of the quality measure provided in step 1. - -<h3>Quality measure</h3> -The text segments matching the query are ranked by (in decreasing order of -significance): - -<ol> - <li> Completeness * keyword weight - - higher ranking if more search words are present in - the same context, and relatively higher weight on matches that - contains "important" terms compared to matches with stop words if - equal number of words. - <li> Proximity - query terms occurring near each other is better - <li> Position - earlier in document is better -</ol> - -The number of matches selected is based on text segment lengths including -a configurable amound of surrounding text, the number of matching segments -to use (configurable) and the required total summary -size (configurable). The final set of matches is -returned with markup for the hits and the abbreviated sections -(continuation symbol). - -The query used for teaser generation has undergone proper name recognition -and English spell checking. Highlighting is done on individual terms of the -query. In particular, phrases are broken down into individual terms, but -the preference to proximal terms will maintain the phrasing in the -generated teaser. - -Lemmatization by expanding documents with word inflections cannot be used -by Juniper. In the future, Juniper would expand the query based on the -original query and language information. This functionality is not -available yet, thus lemmatized terms will in general not be highlighted by -Juniper. - -Currently Juniper uses an alternative, simple brute force stemming -algorithm that basically allows prefixes of the document words to match if -the document word in question is no longer than P (configurable) bytes -<i>longer</i> than the query keyword. -This algorithm works well for keywords of a certain size, but not for very -short keywords. Thus an additional configuration variable defines a lower -bound for what lengths of keywords that will be subject to this algorithm. -With this simple algorithm in place, typical weak form singular to plural -mapping will get highlighted while the opposite, going from a long form to -a shorter one will not work as might be expected. Eg. this algorithm does -not change the keywords themselves. Consequently, the shorter forms of the -keywords are more likely to give non-exact hits in the dynamic summary. - - -<h2>Fast Search configuration of Juniper</h2> -Enabling Juniper functionality within Fast Search is done on a per field -basis by means of override specifications in summary.map. -Currently the following override specs are supported by Juniper: -<p> -<ul> -<li><pre>override <outputfield> dynamicteaser <inputfield></pre> -<li><pre>override <outputfield> dynamicteasermetric -<inputfield></pre> -<li><pre>override <outputfield> juniperlog <inputfield></pre> -</ul> -<p> -Details of the override directive can be found in <i>Fast Search 4.13 - -Dynamic Docsum Generation Framework</i>. -The <tt>dynamicteasermetric</tt> field provides a ranking of the document -based on a corresponding metric as that used to select between individual -matches for dynamic teaser generation inside a document. See the section on -using Juniper for proximity boosting <a href="#proximity">below</a> -The <tt>juniperlog</tt> field is new as of Juniper 2.0, and is used to -retrieve the information generated by Juniper by means of the log query -option, see the runtime option table <a href="#dynpar">below</a>. - -Note that when integrated into Data Search 3.1 and later, -this part of the configuration will be generated via -<ol> -<li>The index profile -<li>The index configuration (indexConfig.xml) by the config server. -</ol> - -<h3>Configuration levels</h3> -When integrated into Fast Search, Juniper receives its default parameters from -global settings in the <dfn>fsearchrc</dfn> config file. -These configuration parameter settings must be preconfigured at -<dfn>fsearch</dfn> process startup time. Two levels of system configuration -is currently supported, -<ol> -<li><b>System default configuration:</b> This is the configuration settings -exemplified by the parameter descriptions below. -<li><b>Per field configuration:</b>By using the field names instead of the -string "juniper" as prefix, the default setting can be overridden on a -<dfn>result field</dfn> basis. Eg. setting for instance -<pre>myfield.dynsum.length 512</pre> (see below) would allow the -<tt>myfield</tt> result field to receive a different teaser length. -</ol> -In addition Juniper 2.x supports changing certain subset of the parameters -on a per query basis. See <a href="#dynpar">separate section</a> on this below. -<p> -<b>Performance note:</b> The per field configuration possibility should be used with -care since overriding some parameters may cause significant computation -overhead in that Juniper would have to scan the whole text multiple times. -Changing the <tt>dynsum</tt> group of fields is generally quite performance -conservative (only the teaser generation phase would have to be repeated), -while changing any of the <tt>stem</tt> or <tt>matcher</tt> -fields would require a different text scan for each combination of -parameters. - -<h3>Arbitrary byte sequences in markup parameters</h3> -To allow arbitrary byte seqences (such as low ascii values) to be used to -denote highlight on/off and continuation symbol(s), Juniper now accepts -strings on the form \xNN where the N's are hex values [0-9a-fA-F]. -This will be converted into a byte value of NN. Note that Juniper exports -UTF-8 text so this sequence should be a valid UTF-8 byte sequence. No -checks are performed from Juniper on the validity of such strings in the -<dfn>fsearch</dfn> domain. -As a consequence of this, occurrences of backslash must be escaped -accordingly (<dfn>\\</dfn>). - - -<h3>Blanks in text parameters must be escaped</h3> -Note that <dfn>fsearchrc</dfn> does not accept blanks in the -parameters. To allow more complicated highlight markup, the sequence -<dfn>\x20</dfn> must be used as space in text fields. - -<p> -<h3><a name="em"></a>Escaping markup in the summary text</h3> -Since Juniper may supply markup through the use of the -highlight and continuation parameters, problems may occur if the -analysed text itself contains markup. To avoid this, Juniper may be -configured to escape the 5 XML/HTML markup symbols -(<dfn>"&<'></dfn>) before adding the mentioned -parametrized symbols. See the description of the -juniper.dynsum.escape_markup parameter <a href="#empar">below</a>. -<p> -The following variables are available for static, global configuration for -a particular search node: -<p> - -<table><a name="conftable"></a> - <tr bgcolor="#f0f0f0"><td><b>Parameter name</b></td><td><b>Default - value</b></td><td><b>Description</b></td> - </tr> - - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.highlight_on</td><td><b></td> - <td>A string to be included <i>before</i> each hit in the generated summary</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.highlight_off</td> - <td></b></td><td>A string to be included <i>after</i> each hit in - the generated summary</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.continuation</td> - <td>...</td><td>A string to be included to denote - abbreviated/left out pieces of the original text in the generated summary</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.separators</td> - <td>\x1D\x1F</td><td>A string containing characters that are added for - word separation purposes (eg.CJK languages and German/Norwegian - etc. word separation). This list should contain non-word characters - only for this to be meaningful. Also, currently only single byte - characters are supported. These characters wil be removed from the - generated teaser by Juniper.</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.connectors</td> - <td>-'</td><td>A string containing characters that may connect two word - tokens to form a single word. Words connected by a single such - character will not be splitted by Juniper when generating the teaser.</td> - <tr bgcolor="#f0f0f0"> - <td><a name="empar"></a>juniper.dynsum.escape_markup</td> - <td>auto</td><td>See <a href="#em">description</a> above. Accepted values: - <dfn>on</dfn>,<dfn>off</dfn> or <dfn>auto</dfn>. If <dfn>auto</dfn> is - used, Juniper will escape markup in the generated summary if any of the symbols - <dfn>highlight_on</dfn>, <dfn>highlight_off</dfn> or - <dfn>continuation</dfn> contains a <dfn><</dfn> as the first - character. - </td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.length</td> - <td>256</td><td>Length of the generated summary in bytes. This is a - hint to Juniper. The result may be slightly longer or shorter depending - on the structure of the available document text and the submitted - query.</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.max_matches</td> - <td>4</td><td>The number of (possibly partial) set of keywords - matching the query, to attempt to include in the summary. The larger this - value compared is set relative to the <i>length</i> parameter, the more - dense the keywords may appear in the summary.</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.min_length</td> - <td>128</td><td>Minimal desired length of the generated summary in - bytes. This is the shortest summary length for which the number of - matches will be respected. Eg. if - a summary appear to become shorter than <i>min_length</i> bytes with - <i>max_matches</i> matches, then additional matches will be used if available.</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.dynsum.surround_max</td> - <td>80</td><td>The maximal number of bytes of context to prepend and append to - each of the selected query keyword hits. This parameter defines the - max size a summary would become if there are few keyword hits - (max_matches set low or document contained few matches of the - keywords.</td> - <tr bgcolor="#f0f0f0"> - <td>juniper.stem.min_length</td> - <td>5</td><td>The minimal number of bytes in a query keyword for - it to be subject to the simple Juniper stemming algorithm. Keywords - that are shorter than or equal to this limit will only yield exact - matches in the dynamic summaries. - </td> - <tr bgcolor="#f0f0f0"> - <td>juniper.stem.max_extend</td> - <td>3</td><td>The maximal number of bytes that a word in the document - can be <i>longer</i> than the keyword itself to yield a match. Eg. for - the default values, if the keyword is 7 bytes long, it will match any - word with length less than or equal to 10 for which the keyword is a prefix. - </td> - </tr> - <tr bgcolor="#f0f0f0"> - <td>juniper.matcher.winsize</td> - <td>400</td><td>The size of the sliding window used to determine if - multiple query terms occur together. The larger the value, the more - likely the system will find (and present in dynamic summary) complete - matches containing all the search terms. The downside is a potential - performance overhead of keeping candidates for matches longer during - matching, and consequently updating more candidates that eventually - gets thrown. - </td> - </tr> - <tr bgcolor="#f0f0f0"> - <td>juniper.proximity.factor</td><td>0.25</td><td> -A factor to multiply the internal Juniper metric with when producing -proximity metric for a given field. A real/floating point value accepted -Note that the QRserver (see <a href="#qrserver">below</a>) -also supports a factor that is global to all proximity -metric fields, and that is applied in addition. </td> - </tr> - -</table> - -<h2><a name="dynpar"></a>Alternate behaviour of Juniper on a per query -basis</h2> -As of Juniper v.2.x and Fastserver v.4.17, and QRserver for Data Search -3.2, Juniper supports a number of Juniper specific options that can be -provided as part of the URL. The format of the option string is -<pre> - juniper=<param_name>.<value>[_<param_name>.<value>]* -</pre> -As an example, consider the following URL addition: -<pre> - juniper=near.2_dynlength.512_dynmatches.8 -</pre> -If this string is present in the URL, Juniper would generate teasers that -are up to twice as long and contains up to twice as many matches of the -query compared to the default values. -In addition, teasers (and <a href="#proximity">proximity metric</a>) will -only be -generated for those documents that fulfills the extra constraint that there -exist at least one complete match of the query where the distance in words -between the first and the last word of the query match is no more than 2 (+ -the number of words in the query). - -<h3>Supported per query options in Juniper 2.1.0</h3> - -<table> - <tr bgcolor="#f0f0f0"><td><b>Parameter - name</b></td><td><b>Corresponding config name, see <a href="#conftable">above</a></b></td><td><b>Description</b></td></tr> - <tr bgcolor="#f0f0f0"><td>dynlength</td><td>dynsum.length</td> - <td>The desired max length of the generated teaser</td></tr> - <tr bgcolor="#f0f0f0"><td>dynmatches</td><td>dynsum.max_matches</td> - <td>The number of matches to try to fit in the teaser</td> - </tr> - <tr bgcolor="#f0f0f0"><td>dynsurmax</td><td>dynsum.surround_max</td> - <td>The maximal amount of surrounding context per keyword hit</td> - </tr> - <tr bgcolor="#f0f0f0"><td>near</td><td><i>N/A</i></td><td>Specifies a - proximity search where keywords should occur closer than the specified - value in number of words not counting the query terms themselves.</td> - </tr> - <tr bgcolor="#f0f0f0"><td>stemext</td><td>juniper.stem.max_extend</td><td> - The maximal number of bytes that a word in the document can be longer - than the keyword itself to yield a match.</td> - </tr> - <tr bgcolor="#f0f0f0"><td>stemmin</td><td>juniper.stem.min_length</td><td> - The minimal number of bytes in a query keyword for it to be subject to - the simple Juniper stemming algorithm.</td> - </tr> - <tr bgcolor="#f0f0f0"><td>within</td><td><i>N/A</i></td><td>Same as - <i>near</i> with the additional constraint that matches of the query must - have the same order of the query words as the original query.</td> - </tr> - <tr bgcolor="#f0f0f0"><td>winsize</td><td>juniper.matcher.winsize</td><td> - The size of the sliding window used to determine if multiple query terms - occur together.</td> - </tr> - <tr bgcolor="#f0f0f0"><td>log</td><td><i>N/A</i></td><td>Internal debug - option (privileged port only). Value is a bitmap that allows selectively - enabled log output to be generated by Juniper for output into a - juniperlog override configured summary field. Useful only with a special - template that makes use of this information. Currently the only - supported bit is 0x8000 which will provide a html table with up to 20 of the - topmost matches of each document, and their identified proximity - (distance) and rank. -</td> -</table> - -<h3>Juniper debug template in Data Search</h3> -Template support for the log parameter as well as extracting the whole -juniper input document text is provided by Data Search 3.2 by means of the -<tt>jsearch</tt> page from the qrserver port. Replace <tt>asearch</tt> -with <tt>jsearch</tt> -in the URL of a qrserver privileged port search. - -<h2>Using Juniper for proximity boosting with the QRserver</h2> -In order to use Juniper to boost hits that have good proximity of the query -(or to filter the hits based on NEAR or WITHIN constraints) the QRserver -would need to be provided with the following URL addition: -<pre> -rpf_proximitybooster:enabled=1 -</pre> -Note that Juniper will return 0 as proximity metric (dynamicteasermetric) -if the query with juniper option constraints cannot be satisfied by -the information in the configured input field. Thus if the selection of a -hit is done solely on the basis of information not present in the Juniper input -(such as the title in the default configuration) -proximity boosting may demote such hits. A solution for this problem has -been proposed for future versions of Juniper. - -<h3><a name="qrserver"></a>Supported QRserver options to use with proximity -boosting via Juniper</h3> -QRserver behaviour wrt. proximity boosting can be set both in configuration -(at QRserver startup) or on a per query basis. In the below table, some of the -default configuration settings are listed together with their corresponding -runtime setting, if any. Consult QRserver documentation for the complete -list of options. -<p> -<table> - <tr bgcolor="#f0f0f0"><td><b>Config parameter - name</b></td><td><b>Corresponding runtime (URL) - syntax</a></b></td><td><b>Description</b></td></tr> - <tr bgcolor="#f0f0f0"><td>rp.proximityboost.enabled=1</td><td>N/A</td><td>Configure for - proximity boosting in the QRserver (not necessarily enable it)</td></tr> - <tr - bgcolor="#f0f0f0"><td>#rp.proximityboost.default</td> - <td>rpf_proximityboost:enabled=1</td><td>Enable proximity - boosting</td></tr> - <tr - bgcolor="#f0f0f0"><td>rp.proximityboost.factor=0.5</td> - <td>N/A</td><td>A value that the combined proximity boost value - calculated possibly from multiple fields, scaled by their individual - factors are multipled with before adding it to the Fastserver rank value - to be used to reorder hits.</td></tr> - <tr bgcolor="#f0f0f0"><td>rp.proximityboost.hits=100</td> - <td>rpf_proximityboost:hits=100</td><td>The number of Fastserver hits to - retrieve as basis for the reordering.</td></tr> - <tr - bgcolor="#f0f0f0"><td>rp.proximityboost.maxoffset=100</td> - <td>N/A</td><td>The maximal offset within the list of hits that will be - subject to any proximity boost reordering/filtering. Hits above this - range in the original result set will not be subject to proximity boosting.</td></tr> -</table> - -<h2>Configuring Juniper within Data Search</h2> -Except where explicitly noted, configuring Juniper for Data Search is -similar to configuring for Real-Time Search. As of Data Search 3.0 Juniper -is by default configured and enabled in Data Search. - -<h2>Configuring Juniper within Fast Real-Time Search</h2> -Juniper is provided as part of Real-Time Search (through Fast Search) -starting with version 2.4. To enable the Fast Search integrated Juniper in -a Real-Time Search environment, see the documentation extensions to -Real-Time Search 2.4. Note that to configure Juniper within Real-Time -Search, the configuration variables should be put in the -<tt>etc/fsearch.addon*</tt> file(s) which will be used as input when Real-Time -Search generates <tt>fsearchrc</tt> files for all configured search -engines. Also a proper <tt>summary.map</tt> file is needed to enable the -dynamic summaries on particular fields. - -<h2>Configuring Juniper for Fast Search v.4.15 and higher</h2> -Newer versions of Fast Search provide template support to allow different -Juniper markup depending on the type of display desired (plain,html or -xml). - -All the http frontends that needs an interpretation of the highlight -information provided by Juniper should have the following setup for Juniper: -<p> -<table> -<tr bgcolor="#f0f0f0"><td>juniper.dynsum.highlight_on</td><td>\02</td></tr> -<tr bgcolor="#f0f0f0"><td>juniper.dynsum.highlight_off</td><td>\03</td></tr> -<tr bgcolor="#f0f0f0"><td>juniper.dynsum.continuation</td><td>\1E</td></tr> -</table> -<p> -The actual frontend markup configuration then takes place by setting -variables such as -<p> -<table> -<tr bgcolor="#f0f0f0"><td>tvm.dynsum.html.highlight_on</td></tr> -<tr bgcolor="#f0f0f0"><td>tvm.dynsum.xml.highlight_off</td></tr> -</table> -<p> -in the relevant rc file. - -<h2>How to report bugs/errors related to Juniper</h2> -Errors/problems related to Juniper can be divided into two categories: -<ol> - <li> Problems with specific documents/teasers - <li> System errors/instability etc. -</ol> -Due to the complexity of a full, running system it is much easier for all -parts if the particular query/document pair triggering the problem can be -identified and analysed off-line. - -<h3>Problems with specific teasers</h3> -Problems of this category is likely to occur because there are so many -combinations of queries and documents that it is not possible to -test for all cases. To be able to analyse such problems, it is vital that -the exact (byte-by-byte) teaser generation source (document summary input -to Juniper) can be made available together with the exact query as -presented to Juniper. To determine this requires the following information: -<ol> - <li> The teaser source docsum. The name of the docsum field is dependent - on the configuration in summary.map. The data should be provided without - any post processing performed, if possible, to avoid missing problems - related to bad input data such as malformet UTF8 characters. - <li> The original query as submitted by the user - <li> The expanded query (available under <tt>var/log/querylogs/</tt>) - <li> A corresponding <tt>fsearchrc</tt> (pure fastserver4) or - <tt>fsearch.addon</tt> (Real Time Search/DS 3.x) file used by the - fsearch process that performed the task. -</ol> - - -<h3>System errors/instability problems</h3> -Problems of category 2 should, if occurring at all only be associated with -development/beta releases. If such an unfortunate event should happen, the -following information <dfn>in addition to the information associated with -category 1</dfn> would be useful to pin down the problem: -<ol> - <li> Core file of <dfn>fsearch</dfn> accompanied with the associated - <dfn>fsearch</dfn> binary. - <li> Log files from the crashed process (in Data Search these will be - present as <tt>var/fsearch-*.log</tt> and <tt>var/log/stdout.log</tt>. -</ol> diff --git a/juniper/src/Doxyfile b/juniper/src/Doxyfile deleted file mode 100644 index fb848bfd931..00000000000 --- a/juniper/src/Doxyfile +++ /dev/null @@ -1,931 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.2.14 - -# This file describes the settings to be used by the documentation system -# doxygen (www.doxygen.org) for a project -# -# All text after a hash (#) is considered a comment and will be ignored -# The format is: -# TAG = value [value, ...] -# For lists items can also be appended using: -# TAG += value [value, ...] -# Values that contain spaces should be placed between quotes (" ") - -#--------------------------------------------------------------------------- -# General configuration options -#--------------------------------------------------------------------------- - -# The PROJECT_NAME tag is a single word (or a sequence of words surrounded -# by quotes) that should identify the project. - -PROJECT_NAME = Juniper - -# The PROJECT_NUMBER tag can be used to enter a project or revision number. -# This could be handy for archiving the generated documentation or -# if some version control system is used. - -PROJECT_NUMBER = - -# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute) -# base path where the generated documentation will be put. -# If a relative path is entered, it will be relative to the location -# where doxygen was started. If left blank the current directory will be used. - -OUTPUT_DIRECTORY = ../doc/doxygen - -# The OUTPUT_LANGUAGE tag is used to specify the language in which all -# documentation generated by doxygen is written. Doxygen will use this -# information to generate all constant output in the proper language. -# The default language is English, other supported languages are: -# Brazilian, Chinese, Croatian, Czech, Danish, Dutch, Finnish, French, -# German, Greek, Hungarian, Italian, Japanese, Korean, Norwegian, Polish, -# Portuguese, Romanian, Russian, Slovak, Slovene, Spanish and Swedish. - -OUTPUT_LANGUAGE = English - -# If the EXTRACT_ALL tag is set to YES doxygen will assume all entities in -# documentation are documented, even if no documentation was available. -# Private class members and static file members will be hidden unless -# the EXTRACT_PRIVATE and EXTRACT_STATIC tags are set to YES - -EXTRACT_ALL = NO - -# If the EXTRACT_PRIVATE tag is set to YES all private members of a class -# will be included in the documentation. - -EXTRACT_PRIVATE = NO - -# If the EXTRACT_STATIC tag is set to YES all static members of a file -# will be included in the documentation. - -EXTRACT_STATIC = YES - -# If the EXTRACT_LOCAL_CLASSES tag is set to YES classes (and structs) -# defined locally in source files will be included in the documentation. -# If set to NO only classes defined in header files are included. - -EXTRACT_LOCAL_CLASSES = NO - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, Doxygen will hide all -# undocumented members of documented classes, files or namespaces. -# If set to NO (the default) these members will be included in the -# various overviews, but no documentation section is generated. -# This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_MEMBERS = YES - -# If the HIDE_UNDOC_CLASSES tag is set to YES, Doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. -# If set to NO (the default) these class will be included in the various -# overviews. This option has no effect if EXTRACT_ALL is enabled. - -HIDE_UNDOC_CLASSES = YES - -# If the BRIEF_MEMBER_DESC tag is set to YES (the default) Doxygen will -# include brief member descriptions after the members that are listed in -# the file and class documentation (similar to JavaDoc). -# Set to NO to disable this. - -BRIEF_MEMBER_DESC = YES - -# If the REPEAT_BRIEF tag is set to YES (the default) Doxygen will prepend -# the brief description of a member or function before the detailed description. -# Note: if both HIDE_UNDOC_MEMBERS and BRIEF_MEMBER_DESC are set to NO, the -# brief descriptions will be completely suppressed. - -REPEAT_BRIEF = YES - -# If the ALWAYS_DETAILED_SEC and REPEAT_BRIEF tags are both set to YES then -# Doxygen will generate a detailed section even if there is only a brief -# description. - -ALWAYS_DETAILED_SEC = NO - -# If the INLINE_INHERITED_MEMB tag is set to YES, doxygen will show all inherited -# members of a class in the documentation of that class as if those members were -# ordinary class members. Constructors, destructors and assignment operators of -# the base classes will not be shown. - -INLINE_INHERITED_MEMB = NO - -# If the FULL_PATH_NAMES tag is set to YES then Doxygen will prepend the full -# path before files name in the file list and in the header files. If set -# to NO the shortest path that makes the file name unique will be used. - -FULL_PATH_NAMES = NO - -# If the FULL_PATH_NAMES tag is set to YES then the STRIP_FROM_PATH tag -# can be used to strip a user defined part of the path. Stripping is -# only done if one of the specified strings matches the left-hand part of -# the path. It is allowed to use relative paths in the argument list. - -STRIP_FROM_PATH = - -# The INTERNAL_DOCS tag determines if documentation -# that is typed after a \internal command is included. If the tag is set -# to NO (the default) then the documentation will be excluded. -# Set it to YES to include the internal documentation. - -INTERNAL_DOCS = NO - -# Setting the STRIP_CODE_COMMENTS tag to YES (the default) will instruct -# doxygen to hide any special comment blocks from generated source code -# fragments. Normal C and C++ comments will always remain visible. - -STRIP_CODE_COMMENTS = YES - -# If the CASE_SENSE_NAMES tag is set to NO then Doxygen will only generate -# file names in lower case letters. If set to YES upper case letters are also -# allowed. This is useful if you have classes or files whose names only differ -# in case and if your file system supports case sensitive file names. Windows -# users are adviced to set this option to NO. - -CASE_SENSE_NAMES = YES - -# If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter -# (but less readable) file names. This can be useful is your file systems -# doesn't support long names like on DOS, Mac, or CD-ROM. - -SHORT_NAMES = NO - -# If the HIDE_SCOPE_NAMES tag is set to NO (the default) then Doxygen -# will show members with their full class and namespace scopes in the -# documentation. If set to YES the scope will be hidden. - -HIDE_SCOPE_NAMES = NO - -# If the VERBATIM_HEADERS tag is set to YES (the default) then Doxygen -# will generate a verbatim copy of the header file for each class for -# which an include is specified. Set to NO to disable this. - -VERBATIM_HEADERS = YES - -# If the SHOW_INCLUDE_FILES tag is set to YES (the default) then Doxygen -# will put list of the files that are included by a file in the documentation -# of that file. - -SHOW_INCLUDE_FILES = NO - -# If the JAVADOC_AUTOBRIEF tag is set to YES then Doxygen -# will interpret the first line (until the first dot) of a JavaDoc-style -# comment as the brief description. If set to NO, the JavaDoc -# comments will behave just like the Qt-style comments (thus requiring an -# explict @brief command for a brief description. - -JAVADOC_AUTOBRIEF = YES - -# If the INHERIT_DOCS tag is set to YES (the default) then an undocumented -# member inherits the documentation from any documented member that it -# reimplements. - -INHERIT_DOCS = YES - -# If the INLINE_INFO tag is set to YES (the default) then a tag [inline] -# is inserted in the documentation for inline members. - -INLINE_INFO = YES - -# If the SORT_MEMBER_DOCS tag is set to YES (the default) then doxygen -# will sort the (detailed) documentation of file and class members -# alphabetically by member name. If set to NO the members will appear in -# declaration order. - -SORT_MEMBER_DOCS = YES - -# If member grouping is used in the documentation and the DISTRIBUTE_GROUP_DOC -# tag is set to YES, then doxygen will reuse the documentation of the first -# member in the group (if any) for the other members of the group. By default -# all members of a group must be documented explicitly. - -DISTRIBUTE_GROUP_DOC = NO - -# The TAB_SIZE tag can be used to set the number of spaces in a tab. -# Doxygen uses this value to replace tabs by spaces in code fragments. - -TAB_SIZE = 8 - -# The GENERATE_TODOLIST tag can be used to enable (YES) or -# disable (NO) the todo list. This list is created by putting \todo -# commands in the documentation. - -GENERATE_TODOLIST = YES - -# The GENERATE_TESTLIST tag can be used to enable (YES) or -# disable (NO) the test list. This list is created by putting \test -# commands in the documentation. - -GENERATE_TESTLIST = YES - -# The GENERATE_BUGLIST tag can be used to enable (YES) or -# disable (NO) the bug list. This list is created by putting \bug -# commands in the documentation. - -GENERATE_BUGLIST = YES - -# This tag can be used to specify a number of aliases that acts -# as commands in the documentation. An alias has the form "name=value". -# For example adding "sideeffect=\par Side Effects:\n" will allow you to -# put the command \sideeffect (or @sideeffect) in the documentation, which -# will result in a user defined paragraph with heading "Side Effects:". -# You can put \n's in the value part of an alias to insert newlines. - -ALIASES = - -# The ENABLED_SECTIONS tag can be used to enable conditional -# documentation sections, marked by \if sectionname ... \endif. - -#ENABLED_SECTIONS = simple \ -# juniper \ -# textproc -# utils - -ENABLED_SECTIONS = juniper - -# The MAX_INITIALIZER_LINES tag determines the maximum number of lines -# the initial value of a variable or define consist of for it to appear in -# the documentation. If the initializer consists of more lines than specified -# here it will be hidden. Use a value of 0 to hide initializers completely. -# The appearance of the initializer of individual variables and defines in the -# documentation can be controlled using \showinitializer or \hideinitializer -# command in the documentation regardless of this setting. - -MAX_INITIALIZER_LINES = 30 - -# Set the OPTIMIZE_OUTPUT_FOR_C tag to YES if your project consists of C sources -# only. Doxygen will then generate output that is more tailored for C. -# For instance some of the names that are used will be different. The list -# of all members will be omitted, etc. - -OPTIMIZE_OUTPUT_FOR_C = NO - -# Set the SHOW_USED_FILES tag to NO to disable the list of files generated -# at the bottom of the documentation of classes and structs. If set to YES the -# list will mention the files that were used to generate the documentation. - -SHOW_USED_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- - -# The QUIET tag can be used to turn on/off the messages that are generated -# by doxygen. Possible values are YES and NO. If left blank NO is used. - -QUIET = NO - -# The WARNINGS tag can be used to turn on/off the warning messages that are -# generated by doxygen. Possible values are YES and NO. If left blank -# NO is used. - -WARNINGS = YES - -# If WARN_IF_UNDOCUMENTED is set to YES, then doxygen will generate warnings -# for undocumented members. If EXTRACT_ALL is set to YES then this flag will -# automatically be disabled. - -WARN_IF_UNDOCUMENTED = YES - -# The WARN_FORMAT tag determines the format of the warning messages that -# doxygen can produce. The string should contain the $file, $line, and $text -# tags, which will be replaced by the file and line number from which the -# warning originated and the warning text. - -WARN_FORMAT = - -# The WARN_LOGFILE tag can be used to specify a file to which warning -# and error messages should be written. If left blank the output is written -# to stderr. - -WARN_LOGFILE = - -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag can be used to specify the files and/or directories that contain -# documented source files. You may enter file names like "myfile.cpp" or -# directories like "/usr/src/myproject". Separate the files or directories -# with spaces. - -# Result processing interface: -# -INPUT = rpinterface.h query.h IJuniperProperties.h dpinterface.h - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank the following patterns are tested: -# *.c *.cc *.cxx *.cpp *.c++ *.java *.ii *.ixx *.ipp *.i++ *.inl *.h *.hh *.hxx *.hpp -# *.h++ *.idl *.odl - -FILE_PATTERNS = *.h \ - *.hpp \ - *.txt - -# The RECURSIVE tag can be used to turn specify whether or not subdirectories -# should be searched for input files as well. Possible values are YES and NO. -# If left blank NO is used. - -RECURSIVE = NO - -# The EXCLUDE tag can be used to specify files and/or directories that should -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used select whether or not files or directories -# that are symbolic links (a Unix filesystem feature) are excluded from the input. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. - -EXCLUDE_PATTERNS = - -# The EXAMPLE_PATH tag can be used to specify one or more files or -# directories that contain example code fragments that are included (see -# the \include command). - -EXAMPLE_PATH = - -# If the value of the EXAMPLE_PATH tag contains directories, you can use the -# EXAMPLE_PATTERNS tag to specify one or more wildcard pattern (like *.cpp -# and *.h) to filter out the source-files in the directories. If left -# blank all files are included. - -EXAMPLE_PATTERNS = - -# If the EXAMPLE_RECURSIVE tag is set to YES then subdirectories will be -# searched for input files to be used with the \include or \dontinclude -# commands irrespective of the value of the RECURSIVE tag. -# Possible values are YES and NO. If left blank NO is used. - -EXAMPLE_RECURSIVE = NO - -# The IMAGE_PATH tag can be used to specify one or more files or -# directories that contain image that are included in the documentation (see -# the \image command). - -IMAGE_PATH = - -# The INPUT_FILTER tag can be used to specify a program that doxygen should -# invoke to filter for each input file. Doxygen will invoke the filter program -# by executing (via popen()) the command <filter> <input-file>, where <filter> -# is the value of the INPUT_FILTER tag, and <input-file> is the name of an -# input file. Doxygen will then use the output that the filter program writes -# to standard output. - -INPUT_FILTER = - -# If the FILTER_SOURCE_FILES tag is set to YES, the input filter (if set using -# INPUT_FILTER) will be used to filter the input files when producing source -# files to browse. - -FILTER_SOURCE_FILES = NO - -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- - -# If the SOURCE_BROWSER tag is set to YES then a list of source files will -# be generated. Documented entities will be cross-referenced with these sources. - -SOURCE_BROWSER = NO - -# Setting the INLINE_SOURCES tag to YES will include the body -# of functions and classes directly in the documentation. - -INLINE_SOURCES = NO - -# If the REFERENCED_BY_RELATION tag is set to YES (the default) -# then for each documented function all documented -# functions referencing it will be listed. - -REFERENCED_BY_RELATION = YES - -# If the REFERENCES_RELATION tag is set to YES (the default) -# then for each documented function all documented entities -# called/used by that function will be listed. - -REFERENCES_RELATION = YES - -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- - -# If the ALPHABETICAL_INDEX tag is set to YES, an alphabetical index -# of all compounds will be generated. Enable this if the project -# contains a lot of classes, structs, unions or interfaces. - -ALPHABETICAL_INDEX = NO - -# If the alphabetical index is enabled (see ALPHABETICAL_INDEX) then -# the COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns -# in which this list will be split (can be a number in the range [1..20]) - -COLS_IN_ALPHA_INDEX = 5 - -# In case all classes in a project start with a common prefix, all -# classes will be put under the same header in the alphabetical index. -# The IGNORE_PREFIX tag can be used to specify one or more prefixes that -# should be ignored while generating the index headers. - -IGNORE_PREFIX = - -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES (the default) Doxygen will -# generate HTML output. - -GENERATE_HTML = YES - -# The HTML_OUTPUT tag is used to specify where the HTML docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `html' will be used as the default path. - -HTML_OUTPUT = - -# The HTML_FILE_EXTENSION tag can be used to specify the file extension for -# each generated HTML page (for example: .htm,.php,.asp). If it is left blank -# doxygen will generate files with .html extension. - -HTML_FILE_EXTENSION = .html - -# The HTML_HEADER tag can be used to specify a personal HTML header for -# each generated HTML page. If it is left blank doxygen will generate a -# standard header. - -HTML_HEADER = - -# The HTML_FOOTER tag can be used to specify a personal HTML footer for -# each generated HTML page. If it is left blank doxygen will generate a -# standard footer. - -HTML_FOOTER = - -# The HTML_STYLESHEET tag can be used to specify a user defined cascading -# style sheet that is used by each HTML page. It can be used to -# fine-tune the look of the HTML output. If the tag is left blank doxygen -# will generate a default style sheet - -HTML_STYLESHEET = - -# If the HTML_ALIGN_MEMBERS tag is set to YES, the members of classes, -# files or namespaces will be aligned in HTML using tables. If set to -# NO a bullet list will be used. - -HTML_ALIGN_MEMBERS = YES - -# If the GENERATE_HTMLHELP tag is set to YES, additional index files -# will be generated that can be used as input for tools like the -# Microsoft HTML help workshop to generate a compressed HTML help file (.chm) -# of the generated HTML documentation. - -GENERATE_HTMLHELP = YES - -# If the GENERATE_HTMLHELP tag is set to YES, the GENERATE_CHI flag -# controls if a separate .chi index file is generated (YES) or that -# it should be included in the master .chm file (NO). - -GENERATE_CHI = YES - -# If the GENERATE_HTMLHELP tag is set to YES, the BINARY_TOC flag -# controls whether a binary table of contents is generated (YES) or a -# normal table of contents (NO) in the .chm file. - -BINARY_TOC = NO - -# The TOC_EXPAND flag can be set to YES to add extra items for group members -# to the contents of the Html help documentation and to the tree view. - -TOC_EXPAND = NO - -# The DISABLE_INDEX tag can be used to turn on/off the condensed index at -# top of each HTML page. The value NO (the default) enables the index and -# the value YES disables it. - -DISABLE_INDEX = NO - -# This tag can be used to set the number of enum values (range [1..20]) -# that doxygen will group on one line in the generated HTML documentation. - -ENUM_VALUES_PER_LINE = 4 - -# If the GENERATE_TREEVIEW tag is set to YES, a side panel will be -# generated containing a tree-like index structure (just like the one that -# is generated for HTML Help). For this to work a browser that supports -# JavaScript and frames is required (for instance Mozilla, Netscape 4.0+, -# or Internet explorer 4.0+). Note that for large projects the tree generation -# can take a very long time. In such cases it is better to disable this feature. -# Windows users are probably better off using the HTML help feature. - -GENERATE_TREEVIEW = NO - -# If the treeview is enabled (see GENERATE_TREEVIEW) then this tag can be -# used to set the initial width (in pixels) of the frame in which the tree -# is shown. - -TREEVIEW_WIDTH = 250 - -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- - -# If the GENERATE_LATEX tag is set to YES (the default) Doxygen will -# generate Latex output. - -GENERATE_LATEX = YES - -# The LATEX_OUTPUT tag is used to specify where the LaTeX docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `latex' will be used as the default path. - -LATEX_OUTPUT = - -# If the COMPACT_LATEX tag is set to YES Doxygen generates more compact -# LaTeX documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_LATEX = NO - -# The PAPER_TYPE tag can be used to set the paper type that is used -# by the printer. Possible values are: a4, a4wide, letter, legal and -# executive. If left blank a4wide will be used. - -PAPER_TYPE = a4wide - -# The EXTRA_PACKAGES tag can be to specify one or more names of LaTeX -# packages that should be included in the LaTeX output. - -EXTRA_PACKAGES = - -# The LATEX_HEADER tag can be used to specify a personal LaTeX header for -# the generated latex document. The header should contain everything until -# the first chapter. If it is left blank doxygen will generate a -# standard header. Notice: only use this tag if you know what you are doing! - -LATEX_HEADER = - -# If the PDF_HYPERLINKS tag is set to YES, the LaTeX that is generated -# is prepared for conversion to pdf (using ps2pdf). The pdf file will -# contain links (just like the HTML output) instead of page references -# This makes the output suitable for online browsing using a pdf viewer. - -PDF_HYPERLINKS = YES - -# If the USE_PDFLATEX tag is set to YES, pdflatex will be used instead of -# plain latex in the generated Makefile. Set this option to YES to get a -# higher quality PDF documentation. - -USE_PDFLATEX = YES - -# If the LATEX_BATCHMODE tag is set to YES, doxygen will add the \\batchmode. -# command to the generated LaTeX files. This will instruct LaTeX to keep -# running if errors occur, instead of asking the user for help. -# This option is also used when generating formulas in HTML. - -LATEX_BATCHMODE = NO - -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- - -# If the GENERATE_RTF tag is set to YES Doxygen will generate RTF output -# The RTF output is optimised for Word 97 and may not look very pretty with -# other RTF readers or editors. - -GENERATE_RTF = NO - -# The RTF_OUTPUT tag is used to specify where the RTF docs will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `rtf' will be used as the default path. - -RTF_OUTPUT = - -# If the COMPACT_RTF tag is set to YES Doxygen generates more compact -# RTF documents. This may be useful for small projects and may help to -# save some trees in general. - -COMPACT_RTF = NO - -# If the RTF_HYPERLINKS tag is set to YES, the RTF that is generated -# will contain hyperlink fields. The RTF file will -# contain links (just like the HTML output) instead of page references. -# This makes the output suitable for online browsing using WORD or other -# programs which support those fields. -# Note: wordpad (write) and others do not support links. - -RTF_HYPERLINKS = NO - -# Load stylesheet definitions from file. Syntax is similar to doxygen's -# config file, i.e. a series of assigments. You only have to provide -# replacements, missing definitions are set to their default value. - -RTF_STYLESHEET_FILE = - -# Set optional variables used in the generation of an rtf document. -# Syntax is similar to doxygen's config file. - -RTF_EXTENSIONS_FILE = - -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- - -# If the GENERATE_MAN tag is set to YES (the default) Doxygen will -# generate man pages - -GENERATE_MAN = YES - -# The MAN_OUTPUT tag is used to specify where the man pages will be put. -# If a relative path is entered the value of OUTPUT_DIRECTORY will be -# put in front of it. If left blank `man' will be used as the default path. - -MAN_OUTPUT = - -# The MAN_EXTENSION tag determines the extension that is added to -# the generated man pages (default is the subroutine's section .3) - -MAN_EXTENSION = - -# If the MAN_LINKS tag is set to YES and Doxygen generates man output, -# then it will generate one additional man file for each entity -# documented in the real man page(s). These additional files -# only source the real man page, but without them the man command -# would be unable to find the correct page. The default is NO. - -MAN_LINKS = NO - -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- - -# If the GENERATE_XML tag is set to YES Doxygen will -# generate an XML file that captures the structure of -# the code including all documentation. Note that this -# feature is still experimental and incomplete at the -# moment. - -GENERATE_XML = NO - -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- - -# If the GENERATE_AUTOGEN_DEF tag is set to YES Doxygen will -# generate an AutoGen Definitions (see autogen.sf.net) file -# that captures the structure of the code including all -# documentation. Note that this feature is still experimental -# and incomplete at the moment. - -GENERATE_AUTOGEN_DEF = NO - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES (the default) Doxygen will -# evaluate all C-preprocessor directives found in the sources and include -# files. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES Doxygen will expand all macro -# names in the source code. If set to NO (the default) only conditional -# compilation will be performed. Macro expansion can be done in a controlled -# way by setting EXPAND_ONLY_PREDEF to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES -# then the macro expansion is limited to the macros specified with the -# PREDEFINED and EXPAND_AS_PREDEFINED tags. - -EXPAND_ONLY_PREDEF = NO - -# If the SEARCH_INCLUDES tag is set to YES (the default) the includes files -# in the INCLUDE_PATH (see below) will be search if a #include is found. - -SEARCH_INCLUDES = YES - -# The INCLUDE_PATH tag can be used to specify one or more directories that -# contain include files that are not input files but should be processed by -# the preprocessor. - -INCLUDE_PATH = - -# You can use the INCLUDE_FILE_PATTERNS tag to specify one or more wildcard -# patterns (like *.h and *.hpp) to filter out the header-files in the -# directories. If left blank, the patterns specified with FILE_PATTERNS will -# be used. - -INCLUDE_FILE_PATTERNS = - -# The PREDEFINED tag can be used to specify one or more macro names that -# are defined before the preprocessor is started (similar to the -D option of -# gcc). The argument of the tag is a list of macros of the form: name -# or name=definition (no spaces). If the definition and the = are -# omitted =1 is assumed. - -PREDEFINED = - -# If the MACRO_EXPANSION and EXPAND_PREDEF_ONLY tags are set to YES then -# this tag can be used to specify a list of macro names that should be expanded. -# The macro definition that is found in the sources will be used. -# Use the PREDEFINED tag if you want to use a different macro definition. - -EXPAND_AS_DEFINED = - -# If the SKIP_FUNCTION_MACROS tag is set to YES (the default) then -# doxygen's preprocessor will remove all function-like macros that are alone -# on a line and do not end with a semicolon. Such function macros are typically -# used for boiler-plate code, and will confuse the parser if not removed. - -SKIP_FUNCTION_MACROS = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to external references -#--------------------------------------------------------------------------- - -# The TAGFILES tag can be used to specify one or more tagfiles. - -TAGFILES = ../../fastos/doc/doxygen/fastos.tag - -# When a file name is specified after GENERATE_TAGFILE, doxygen will create -# a tag file that is based on the input files it reads. - -GENERATE_TAGFILE = juniper.tag - -# If the ALLEXTERNALS tag is set to YES all external classes will be listed -# in the class index. If set to NO only the inherited external classes -# will be listed. - -ALLEXTERNALS = NO - -# If the EXTERNAL_GROUPS tag is set to YES all external groups will be listed -# in the modules index. If set to NO, only the current project's groups will -# be listed. - -EXTERNAL_GROUPS = YES - -# The PERL_PATH should be the absolute path and name of the perl script -# interpreter (i.e. the result of `which perl'). - -PERL_PATH = - -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- - -# If the CLASS_DIAGRAMS tag is set to YES (the default) Doxygen will -# generate a inheritance diagram (in Html, RTF and LaTeX) for classes with base or -# super classes. Setting the tag to NO turns the diagrams off. Note that this -# option is superceded by the HAVE_DOT option below. This is only a fallback. It is -# recommended to install and use dot, since it yield more powerful graphs. - -CLASS_DIAGRAMS = YES - -# If you set the HAVE_DOT tag to YES then doxygen will assume the dot tool is -# available from the path. This tool is part of Graphviz, a graph visualization -# toolkit from AT&T and Lucent Bell Labs. The other options in this section -# have no effect if this option is set to NO (the default) - -HAVE_DOT = NO - -# If the CLASS_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect inheritance relations. Setting this tag to YES will force the -# the CLASS_DIAGRAMS tag to NO. - -CLASS_GRAPH = NO - -# If the COLLABORATION_GRAPH and HAVE_DOT tags are set to YES then doxygen -# will generate a graph for each documented class showing the direct and -# indirect implementation dependencies (inheritance, containment, and -# class references variables) of the class with other documented classes. - -COLLABORATION_GRAPH = YES - -# If set to YES, the inheritance and collaboration graphs will show the -# relations between templates and their instances. - -TEMPLATE_RELATIONS = YES - -# If set to YES, the inheritance and collaboration graphs will hide -# inheritance and usage relations if the target is undocumented -# or is not a class. - -HIDE_UNDOC_RELATIONS = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDE_GRAPH, and HAVE_DOT -# tags are set to YES then doxygen will generate a graph for each documented -# file showing the direct and indirect include dependencies of the file with -# other documented files. - -INCLUDE_GRAPH = YES - -# If the ENABLE_PREPROCESSING, SEARCH_INCLUDES, INCLUDED_BY_GRAPH, and -# HAVE_DOT tags are set to YES then doxygen will generate a graph for each -# documented header file showing the documented files that directly or -# indirectly include this file. - -INCLUDED_BY_GRAPH = YES - -# If the GRAPHICAL_HIERARCHY and HAVE_DOT tags are set to YES then doxygen -# will graphical hierarchy of all classes instead of a textual one. - -GRAPHICAL_HIERARCHY = YES - -# The DOT_IMAGE_FORMAT tag can be used to set the image format of the images -# generated by dot. Possible values are gif, jpg, and png -# If left blank gif will be used. - -DOT_IMAGE_FORMAT = gif - -# The tag DOT_PATH can be used to specify the path where the dot tool can be -# found. If left blank, it is assumed the dot tool can be found on the path. - -DOT_PATH = - -# The DOTFILE_DIRS tag can be used to specify one or more directories that -# contain dot files that are included in the documentation (see the -# \dotfile command). - -DOTFILE_DIRS = - -# The MAX_DOT_GRAPH_WIDTH tag can be used to set the maximum allowed width -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_WIDTH = 1024 - -# The MAX_DOT_GRAPH_HEIGHT tag can be used to set the maximum allows height -# (in pixels) of the graphs generated by dot. If a graph becomes larger than -# this value, doxygen will try to truncate the graph, so that it fits within -# the specified constraint. Beware that most browsers cannot cope with very -# large images. - -MAX_DOT_GRAPH_HEIGHT = 1024 - -# If the GENERATE_LEGEND tag is set to YES (the default) Doxygen will -# generate a legend page explaining the meaning of the various boxes and -# arrows in the dot generated graphs. - -GENERATE_LEGEND = YES - -# If the DOT_CLEANUP tag is set to YES (the default) Doxygen will -# remove the intermedate dot files that are used to generate -# the various graphs. - -DOT_CLEANUP = YES - -#--------------------------------------------------------------------------- -# Configuration::addtions related to the search engine -#--------------------------------------------------------------------------- - -# The SEARCHENGINE tag specifies whether or not a search engine should be -# used. If set to NO the values of all tags below this one will be ignored. - -SEARCHENGINE = NO - -# The CGI_NAME tag should be the name of the CGI script that -# starts the search engine (doxysearch) with the correct parameters. -# A script with this name will be generated by doxygen. - -CGI_NAME = search.cgi - -# The CGI_URL tag should be the absolute URL to the directory where the -# cgi binaries are located. See the documentation of your http daemon for -# details. - -CGI_URL = - -# The DOC_URL tag should be the absolute URL to the directory where the -# documentation is located. If left blank the absolute path to the -# documentation, with file:// prepended to it, will be used. - -DOC_URL = - -# The DOC_ABSPATH tag should be the absolute path to the directory where the -# documentation is located. If left blank the directory on the local machine -# will be used. - -DOC_ABSPATH = - -# The BIN_ABSPATH tag must point to the directory where the doxysearch binary -# is installed. - -BIN_ABSPATH = - -# The EXT_DOC_PATHS tag can be used to specify one or more paths to -# documentation generated for other projects. This allows doxysearch to search -# the documentation for these projects as well. - -EXT_DOC_PATHS = diff --git a/juniper/src/test/matchobjectTestApp.cpp b/juniper/src/test/matchobjectTestApp.cpp deleted file mode 100644 index 7fae5b4c0de..00000000000 --- a/juniper/src/test/matchobjectTestApp.cpp +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "matchobjectTest.h" -#include "testenv.h" -#include <vespa/vespalib/testkit/testapp.h> - -int main(int argc, char **argv) { - juniper::TestEnv te(argc, argv, TEST_PATH("../rpclient/testclient.rc").c_str()); - MatchObjectTest test; - test.SetStream(&std::cout); - test.Run(argc, argv); - return (int)test.Report(); -} diff --git a/juniper/src/testproject.el b/juniper/src/testproject.el deleted file mode 100644 index fa8d580b0eb..00000000000 --- a/juniper/src/testproject.el +++ /dev/null @@ -1,98 +0,0 @@ -;; Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -;; testproject.el - -;; Local configurations for the cpptest Emacs unit-test -;; framework. This is just an example of typical variables that one -;; usually uses This file should be located in the same directory as -;; the class(es) you want to test. - -;; $Revision: 1.2 $ $Date: 2003-02-27 12:32:24 $ -;; Author: Nils Sandøy <nils.sandoy@fast.no> - -;; Just a message to show that this file is beeing read. Look for this -;; in the *Messages* buffer. -(message "Setting local test configuration for the module") - -;; Don't use an underscore based naming scheme -;; classes and method names will Upcase each word instead -(setq cppt-use-underscore-p nil) - -;; Use author in documentation. Set this value to nil if not -(setq cppt-doc-author-p "t") - -;; This is a subdirectory of the directory in which this file, along with -;; the source code to test, resides -(setq cppt-test-dir "test") - -;; Use this variable to include extra file in your test source, and -;; application files. Typically this will hold headers for log -;; functionality etc. -;; Example: (setq cppt-extra-source-includes "#include \"../Log.h\"") -(setq cppt-extra-source-includes "#include \"testenv.h\"") - -;; If the above source files are not part of a library, you will -;; probably have to include them in the fastos.project file. -;; Example: (setq cppt-extra-object-files '("../Log")) -(setq cppt-extra-object-files '("testenv")) - -;; If the source code does not have a fastos.project file with all -;; required libraries for linking an executable (typically the case -;; when the source is part of a library itself), then you should use -;; this variable to provide a list of libraries which will be appended -;; to the EXTERNALLIBS section for all applications in the -;; fastos.project file. -;; Example: (setq cppt-extra-libraries '("fast")) -(setq cppt-extra-libraries '("src/juniper")) -(setq cppt-extra-external-libraries '("fast")) - -;; Include source file in test executables. -;; Set this to nil if you are testing part of a library -(setq cppt-include-source-p "t") - - -;; If your initialisation code below requires special parameters for -;; running the test executables, add them here -;; Example: (setq cppt-test-parameters "--test-mode") -(setq cppt-test-parameters "") - -;; If you support a special debug mode, which is executed through the -;; cppt-suite-debug or cppt-run-test-debug methods, then you should -;; add the parameter for identifying this here -;; Example: (setq cppt-test-dbflags "-d") -(setq cppt-test-dbflags "") - -;; If you support logging etc, you should include code here for -;; insitializing this as part of the Main body of the test application -;; Example: -;; Add intialization code that turns on logging, and logs to stderr in debug -;; mode -;; (setq cppt-application-init-code -;; "RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(\"CLASSTest.log\"), FLOG_ALL); -;; for (int i=0; i < argc; ++i) { -;; if (strcmp(argv[i], \"-d\") == 0) { -;; // Turn on debug mode (log to stderr) -;; RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(stderr), FLOG_ALL); -;; LOG_DBG(\"Running in debug mode\"); -;; } -;; }") -(setq cppt-application-init-code "") - -;; Pretty much the same as the application init code, but this is used -;; for the Main method of the test suite. -;; Example: -;; Add intialization code that turns on logging, and logs to stderr in debug -;; mode -;; (setq cppt-suite-init-code -;; "RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(\"SUITETest.log\"), FLOG_ALL); -;; for (int i=0; i < argc; ++i) { -;; if (strcmp(argv[i], \"-d\") == 0) { -;; // Turn on debug mode (log to stderr) -;; RTLogDistributor::GetInstance().RegisterDestination( -;; new Fast_FileLogger(stderr), FLOG_ALL); -;; LOG_DBG(\"Running in debug mode\"); -;; } -;; }") -(setq cppt-suite-init-code "") diff --git a/linguistics-components/CMakeLists.txt b/linguistics-components/CMakeLists.txt index 0e853b1daac..a26095a4649 100644 --- a/linguistics-components/CMakeLists.txt +++ b/linguistics-components/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(linguistics-components) +install_jar(linguistics-components-jar-with-dependencies.jar) install_config_definitions() diff --git a/logd/CMakeLists.txt b/logd/CMakeLists.txt index 498efc36cf2..d02b99a393a 100644 --- a/logd/CMakeLists.txt +++ b/logd/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_define_module( fastos vespalog vespalib - staging_vespalib config_cloudconfig APPS diff --git a/logd/src/logd/state_reporter.h b/logd/src/logd/state_reporter.h index 1b00a2fe662..eef5df3fce9 100644 --- a/logd/src/logd/state_reporter.h +++ b/logd/src/logd/state_reporter.h @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/vespalib/net/state_server.h> -#include <vespa/vespalib/net/simple_health_producer.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> -#include <vespa/vespalib/net/generic_state_handler.h> +#include <vespa/vespalib/net/http/state_server.h> +#include <vespa/vespalib/net/http/simple_health_producer.h> +#include <vespa/vespalib/net/http/simple_component_config_producer.h> +#include <vespa/vespalib/net/http/generic_state_handler.h> #include <vespa/vespalib/metrics/metrics_manager.h> #include <vespa/vespalib/metrics/producer.h> diff --git a/logserver/CMakeLists.txt b/logserver/CMakeLists.txt index 76f726f8b38..f336bd7af94 100644 --- a/logserver/CMakeLists.txt +++ b/logserver/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(logserver) +install_jar(logserver-jar-with-dependencies.jar) vespa_install_script(bin/logserver-start.sh vespa-logserver-start bin) diff --git a/messagebus/CMakeLists.txt b/messagebus/CMakeLists.txt index 87cbcaa7f01..30e795cac1d 100644 --- a/messagebus/CMakeLists.txt +++ b/messagebus/CMakeLists.txt @@ -5,9 +5,8 @@ vespa_define_module( vespalog config_cloudconfig vespalib - staging_vespalib fnet - slobrok_slobrokserver + slobrok LIBS src/vespa/messagebus diff --git a/messagebus/src/vespa/messagebus/replygate.cpp b/messagebus/src/vespa/messagebus/replygate.cpp index 1b37a542d49..d1bd6ef05c7 100644 --- a/messagebus/src/vespa/messagebus/replygate.cpp +++ b/messagebus/src/vespa/messagebus/replygate.cpp @@ -22,13 +22,13 @@ ReplyGate::handleMessage(Message::UP msg) void ReplyGate::close() { - _open = false; + _open.store(false, std::memory_order_relaxed); } void ReplyGate::handleReply(Reply::UP reply) { - if (_open) { + if (_open.load(std::memory_order_relaxed)) { IReplyHandler &handler = reply->getCallStack().pop(*reply); handler.handleReply(std::move(reply)); } else { diff --git a/messagebus/src/vespa/messagebus/replygate.h b/messagebus/src/vespa/messagebus/replygate.h index d432afa4b8b..0c487de3ecf 100644 --- a/messagebus/src/vespa/messagebus/replygate.h +++ b/messagebus/src/vespa/messagebus/replygate.h @@ -6,6 +6,7 @@ #include "imessagehandler.h" #include "ireplyhandler.h" #include <vespa/vespalib/util/referencecounter.h> +#include <atomic> namespace mbus { @@ -26,7 +27,7 @@ class ReplyGate : public vespalib::ReferenceCounter, { private: IMessageHandler &_sender; - bool _open; + std::atomic<bool> _open; public: /** diff --git a/metrics-proxy/CMakeLists.txt b/metrics-proxy/CMakeLists.txt index c54bbb57cf8..5963cba9615 100644 --- a/metrics-proxy/CMakeLists.txt +++ b/metrics-proxy/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(metrics-proxy) +install_jar(metrics-proxy-jar-with-dependencies.jar) vespa_install_script(src/main/sh/start-telegraf.sh libexec/vespa) vespa_install_script(src/main/sh/stop-telegraf.sh libexec/vespa) diff --git a/metrics/CMakeLists.txt b/metrics/CMakeLists.txt index c5dcf89c02f..d75c17d4a00 100644 --- a/metrics/CMakeLists.txt +++ b/metrics/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_define_module( fastos vespalog vespalib - staging_vespalib config_cloudconfig LIBS diff --git a/metrics/src/vespa/metrics/state_api_adapter.h b/metrics/src/vespa/metrics/state_api_adapter.h index b097727e974..c78d302a820 100644 --- a/metrics/src/vespa/metrics/state_api_adapter.h +++ b/metrics/src/vespa/metrics/state_api_adapter.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/metrics_producer.h> +#include <vespa/vespalib/net/http/metrics_producer.h> namespace metrics { diff --git a/model-evaluation/CMakeLists.txt b/model-evaluation/CMakeLists.txt index ebda5698957..dafccd28e2b 100644 --- a/model-evaluation/CMakeLists.txt +++ b/model-evaluation/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(model-evaluation) +install_jar(model-evaluation-jar-with-dependencies.jar) diff --git a/model-integration/CMakeLists.txt b/model-integration/CMakeLists.txt index 4225ac38f89..4423746ccbc 100644 --- a/model-integration/CMakeLists.txt +++ b/model-integration/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(model-integration) +install_jar(model-integration-jar-with-dependencies.jar) vespa_install_script(src/main/python/vespa-convert-tf2onnx.py vespa-convert-tf2onnx bin) diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java index 367b840a728..1003ebd8f3f 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/configmodelview/ImportedMlModels.java @@ -3,6 +3,7 @@ package ai.vespa.rankingexpression.importer.configmodelview; import com.yahoo.concurrent.InThreadExecutorService; import com.yahoo.path.Path; +import com.yahoo.yolean.Exceptions; import java.io.File; import java.util.Arrays; @@ -55,7 +56,7 @@ public class ImportedMlModels { models.put(name, model); } } catch (InterruptedException | ExecutionException e) { - skippedModels.put(name, e.getMessage()); + skippedModels.put(name, Exceptions.toMessageString(e)); } }); importedModels = Collections.unmodifiableMap(models); @@ -97,7 +98,7 @@ public class ImportedMlModels { if (existing != null) { try { throw new IllegalArgumentException("The models in " + child + " and " + existing.get().source() + - " both resolve to the model name '" + name + "'"); + " both resolve to the model name '" + name + "'"); } catch (InterruptedException | ExecutionException e) {} } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java index 675e18da637..117d699c3a9 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/operations/OnnxConstant.java @@ -84,7 +84,7 @@ public class OnnxConstant extends IntermediateOperation { } if (value.isEmpty()) { throw new IllegalArgumentException("Node '" + name + "' of type " + - "constant has missing or non-supported 'value' attribute"); + "constant has missing or non-supported 'value' attribute"); } return value.get(); } diff --git a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java index cb1dede26d9..5316416c4dc 100644 --- a/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java +++ b/model-integration/src/main/java/ai/vespa/rankingexpression/importer/tensorflow/TensorFlowImporter.java @@ -76,8 +76,8 @@ public class TensorFlowImporter extends ModelImporter { log.fine("Conversion to ONNX with opset " + opset + " failed. Reason: " + res.getSecond()); outputOfLastConversionAttempt = res.getSecond(); } - throw new IllegalArgumentException("Unable to convert TensorFlow model in '" + modelDir + "' to ONNX. " + - "Reason: " + outputOfLastConversionAttempt); + throw new IllegalArgumentException("Unable to convert TensorFlow model in '" + modelDir + "' to ONNX: " + + outputOfLastConversionAttempt); } catch (IOException e) { throw new IllegalArgumentException("Conversion from TensorFlow to ONNX failed for '" + modelDir + "'"); } finally { diff --git a/model-integration/src/main/javacc/ModelParser.jj b/model-integration/src/main/javacc/ModelParser.jj index 6f6f3508beb..c850d223612 100644 --- a/model-integration/src/main/javacc/ModelParser.jj +++ b/model-integration/src/main/javacc/ModelParser.jj @@ -30,8 +30,11 @@ import java.util.List; import java.util.ArrayList; import ai.vespa.rankingexpression.importer.ImportedModel; import com.yahoo.io.IOUtils; -import com.yahoo.tensor.TensorType; import com.yahoo.tensor.Tensor; +import com.yahoo.tensor.IndexedTensor; +import com.yahoo.tensor.MixedTensor; +import com.yahoo.tensor.TensorAddress; +import com.yahoo.tensor.TensorType; import com.yahoo.tensor.serialization.JsonFormat; import com.yahoo.searchlib.rankingexpression.RankingExpression; @@ -80,8 +83,7 @@ TOKEN : { < NL: "\n" > | < FUNCTION: "function" > -| < TENSOR_TYPE: "tensor(" (~["(",")"])+ ")" > -| < TENSORVALUE: (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > +| < TENSOR_TYPE: "tensor" ("<" (~["<",">"])+ ">")? "(" (~["(",")"])* ")" > | < TENSOR_VALUE_SL: "value" (" ")* ":" (" ")* ("{"<BRACE_SL_LEVEL_1>) ("\n")? > | < TENSOR_VALUE_ML: "value" (<SEARCHLIB_SKIP>)? "{" (["\n"," "])* ("{"<BRACE_ML_LEVEL_1>) (["\n"," "])* "}" ("\n")? > | < LBRACE: "{" > @@ -89,6 +91,8 @@ TOKEN : | < COLON: ":" > | < DOT: "." > | < COMMA: "," > +| < DOUBLE_KEYWORD: "double" > +| < INPUTS: "inputs" > | < MODEL: "model" > | < TYPE: "type" > | < EXPRESSION_SL: "expression" (" ")* ":" (("{"<BRACE_SL_LEVEL_1>)|<BRACE_SL_CONTENT>)* ("\n")? > @@ -107,8 +111,12 @@ TOKEN : | < FILE: "file" > | < URI: "uri" > | < IDENTIFIER: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_"])* > +| < DOUBLEQUOTEDSTRING: "\"" ( ~["\""] )* "\"" > +| < SINGLEQUOTEDSTRING: "'" ( ~["'"] )* "'" > | < CONTEXT: ["a"-"z","A"-"Z"] (["a"-"z", "A"-"Z", "0"-"9"])* > | < DOUBLE: ("-")? (["0"-"9"])+ "." (["0"-"9"])+ > +| < INTEGER: ("-")? (["0"-"9"])+ > +| < LONG: ("-")? (["0"-"9"])+"L" > | < STRING: (["a"-"z","A"-"Z","_","0"-"9","."])+ > | < FILE_PATH: ["a"-"z","A"-"Z", "_"] (["a"-"z","A"-"Z","0"-"9","_","-", "/", "."])+ > | < HTTP: ["h","H"] ["t","T"] ["t","T"] ["p","P"] (["s","S"])? > @@ -149,10 +157,23 @@ void model() : } void modelContent() : +{} { + ( <NL> | + constants() | + largeConstant() | + function() | + inputs() | + input() + )* } + +void inputs() : +{} { - ( <NL> | input() | constants() | largeConstant() | function() )* + <INPUTS> (<NL>)* <LBRACE> (<NL>)* + ( input() (<NL>)* )* + <RBRACE> } /** Declared input variables (aka features). All non-scalar inputs must be declared. */ @@ -191,36 +212,244 @@ void constants() : } { <CONSTANTS> <LBRACE> (<NL>)* - ( name = identifier() <COLON> ( constantDouble(name) | constantTensor(name) ) (<NL>)* )* + ( constant() (<NL>)* )* <RBRACE> } -void constantDouble(String name) : +String constantTensorErrorMessage(String constantTensorName) : {} +{ + { return "For constant tensor '" + constantTensorName + "' in '" + model + "'"; } +} + +void constant() : +{ + String name = null; + TensorType type = TensorType.empty; + Tensor value = null; + String valuePath = null; +} +{ + ( + name = identifier() (<COLON>)? + ( + LOOKAHEAD(4) ( ( type = valueType(name) )? (<COLON>)? (<NL>)* ( value = tensorValue(type) | valuePath = fileItem()) + { + if (value != null) { + model.smallConstant(name, value); + } + else { + try { + value = JsonFormat.decode(type, IOUtils.readFileBytes(model.relativeFile(valuePath, "constant '" + name + "'"))); + model.largeConstant(name, value); + } + catch (Exception e) { + throw new IllegalArgumentException("Could not read constant '" + name + "'", e); + } + } + } + ) + | // Deprecated forms (TODO: Add warning on Vespa 8): + ( constantValue(name) | constantTensor(name) ) + ) + ) +} + +// Deprecated form +void constantValue(String name) : { Token value; } { - value = <DOUBLE> { model.smallConstant(name, Tensor.from(Double.parseDouble(value.image))); } + <COLON> ( value = <DOUBLE> | value = <INTEGER> | value = <IDENTIFIER> ) + { model.smallConstant(name, Tensor.from(value.image)); } } +// Deprecated form void constantTensor(String name) : { + String tensorString = ""; + TensorType type = null; +} +{ + <LBRACE> (<NL>)* + (( tensorString = tensorValuePrefixedByValue() | + type = tensorTypeWithPrefix(constantTensorErrorMessage(name)) ) (<NL>)* )* <RBRACE> + { model.smallConstant(name, type != null ? Tensor.from(type, tensorString) : Tensor.from(tensorString)); } +} + +TensorType valueType(String name) : +{ TensorType type; - Token value; + } { - type = tensorType("constant '" + name + "'") value = <TENSORVALUE> - { - model.smallConstant(name, Tensor.from(type, value.image.substring(1))); - } + ( + ( type = tensorType("Type of " + name) ) + | + ( <DOUBLE_KEYWORD> { type = TensorType.empty; } ) + ) + { return type; } } -String constantTensorErrorMessage(String model, String constantTensorName) : {} +TensorType tensorType(String errorMessage) : { - { return "For constant tensor '" + constantTensorName + "' in model '" + model + "'"; } + String tensorTypeString; +} +{ + <TENSOR_TYPE> { tensorTypeString = token.image; } + { + TensorType tensorType; + try { + tensorType = TensorType.fromSpec(tensorTypeString); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException(errorMessage + ": Illegal tensor type spec: " + e.getMessage()); + } + return tensorType; + } } -String tensorValue() : +/** + * Parses a tensor written in a tensor literal form, + * https://docs.vespa.ai/en/reference/tensor.html#tensor-literal-form + */ +Tensor tensorValue(TensorType type) : +{ + Tensor.Builder builder = Tensor.Builder.of(type); + Number doubleValue = null; +} +{ + ( mappedTensorValue(builder) | indexedTensorValues(builder) | doubleValue = number() ) + { + if (doubleValue != null) { + if (type.rank() > 0) + throw new IllegalArgumentException("A tensor of type " + type + " cannot be a number"); + builder.cell(doubleValue.doubleValue()); + } + return builder.build(); + } +} + +/** A mapped or mixed tensor value. */ +void mappedTensorValue(Tensor.Builder builder) : {} +{ + "{" + ( mappedTensorBlock(builder) )* + ( <COMMA> (<NL>)* mappedTensorBlock(builder) )* + "}" +} + + +void mappedTensorBlock(Tensor.Builder builder) : +{ + TensorAddress mappedAddress; +} +{ + mappedAddress = tensorAddress(builder.type()) <COLON> (<NL>)* + ( mappedTensorCellValue(mappedAddress, builder) | indexedTensorBlockValues(mappedAddress, builder) ) +} + +void indexedTensorBlockValues(TensorAddress mappedAddress, Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + MixedTensor.BoundBuilder boundBuilder = (MixedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.block(mappedAddress, arrayValues); + } +} + +void indexedTensorValues(Tensor.Builder builder) : +{ + List<Double> values = new ArrayList<Double>(); +} +{ + arrayTensorValues(values) + { + IndexedTensor.BoundBuilder boundBuilder = (IndexedTensor.BoundBuilder)builder; + double[] arrayValues = new double[values.size()]; + for (int i = 0; i < values.size(); i++ ) { + arrayValues[i] = values.get(i); + } + boundBuilder.fill(arrayValues); + } +} + +/** Tensor array values. Using sub-bracketing for multiple dimensions is optional and therefore ignored here. */ +void arrayTensorValues(List<Double> values) : {} +{ + "[" ( ( indexedTensorValue(values) | arrayTensorValues(values)) )* + ( <COMMA> (<NL>)* ( indexedTensorValue(values) | arrayTensorValues(values)) )* + "]" +} + +void indexedTensorValue(List<Double> values) : +{ + Number value; +} +{ + value = number() + { values.add(value.doubleValue()); } +} + +void mappedTensorCellValue(TensorAddress address, Tensor.Builder builder) : +{ + double value; +} +{ + value = tensorCellValue() + { builder.cell(address, value); } +} + +TensorAddress tensorAddress(TensorType type) : +{ + TensorAddress.Builder builder = new TensorAddress.PartialBuilder(type); + String label; +} +{ + ( + label = tensorAddressLabel() { builder.add(label); } + | + ( "{" ( tensorAddressElement(builder) )* ( <COMMA> tensorAddressElement(builder) )* "}" ) + ) + { return builder.build(); } +} + +void tensorAddressElement(TensorAddress.Builder builder) : +{ + String dimension; + String label; +} +{ + dimension = identifier() <COLON> (<NL>)* label = tensorAddressLabel() + { builder.add(dimension, label); } +} + +String tensorAddressLabel() : +{ + String label; +} +{ + ( label = identifier() | label = quotedString() ) + { return label; } +} + +double tensorCellValue() : +{ + Number value; +} +{ + value = number() + { return value.doubleValue(); } +} + +/** Undocumented syntax for supplying a tensor constant value by a string prefixed by "value" */ +String tensorValuePrefixedByValue() : { String tensor; } @@ -233,7 +462,7 @@ String tensorValue() : } } -TensorType tensorType(String errorMessage) : +TensorType tensorTypeWithPrefix(String errorMessage) : { String tensorTypeString; } @@ -250,7 +479,7 @@ TensorType tensorType(String errorMessage) : } } -/** Consumes a large constant. */ +/** Consumes a large constant. */ // TODO: Remove on Vespa 9 void largeConstant() : { String name; @@ -311,18 +540,65 @@ String expression() : String identifier() : { } { ( - <IDENTIFIER> - | <DOUBLE> - | <FILE> - | <URI> - | <MODEL> - | <TYPE> + <CONSTANT> | + <CONSTANTS> | + <DOUBLE_KEYWORD> | + <FILE> | + <IDENTIFIER> | + <INPUTS> | + <INTEGER> | + <MODEL> | + <TYPE> | + <URI> ) { return token.image; } } +Number number() : +{ + Number num; +} +{ + (num = floatValue() | num = longValue() ) { return num; } +} + +/** Consumes a long or integer token and returns its numeric value. */ +long longValue() : { } +{ + ( <INTEGER> { return Long.parseLong(token.image); } | + <LONG> { return Long.parseLong(token.image.substring(0, token.image.length()-1)); } + ) +} + +/** Consumes a floating-point token and returns its numeric value. */ +double floatValue() : { } +{ + <DOUBLE> { return Double.valueOf(token.image); } +} + /** Consumes an opening brace with leading and trailing newline tokens. */ void lbrace() : { } { (<NL>)* <LBRACE> (<NL>)* } + +String fileItem() : +{ + String path; +} +{ + (<FILE> <COLON> ( <FILE_PATH> | <STRING> | <IDENTIFIER>) { path = com.yahoo.path.Path.fromString(token.image).getRelative(); } { } (<NL>)*) { return path; } +} + +/** + * Consumes a quoted string token and returns the token image minus the quotes. This does not perform + * unescaping of the content, it simply removes the first and last character of the image. However, the token itself can + * contain anything but a double quote. + * + * @return the unquoted token image + */ +String quotedString() : { } +{ + ( <DOUBLEQUOTEDSTRING> | <SINGLEQUOTEDSTRING> ) + { return token.image.substring(1, token.image.length() - 1); } +} diff --git a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java index fc92883a90f..25c51a75b0b 100644 --- a/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java +++ b/model-integration/src/test/java/ai/vespa/rankingexpression/importer/vespa/VespaImportTestCase.java @@ -25,7 +25,16 @@ public class VespaImportTestCase { @Test public void testExample() { ImportedModel model = importModel("example"); + assertModel(model); + } + + @Test + public void testLegacySyntax() { + ImportedModel model = importModel("legacy_syntax"); + assertModel(model); + } + private void assertModel(ImportedModel model) { assertEquals(2, model.inputs().size()); assertEquals("tensor(name{},x[3])", model.inputs().get("input1").toString()); assertEquals("tensor(x[3])", model.inputs().get("input2").toString()); diff --git a/model-integration/src/test/models/vespa/example.model b/model-integration/src/test/models/vespa/example.model index 269ed83b695..25d27033cfd 100644 --- a/model-integration/src/test/models/vespa/example.model +++ b/model-integration/src/test/models/vespa/example.model @@ -1,17 +1,15 @@ model example { # All inputs that are not scalar (aka 0-dimensional tensor) must be declared - input1: tensor(name{}, x[3]) - input2: tensor(x[3]) + inputs { + input1: tensor(name{}, x[3]) + input2: tensor(x[3]) + } constants { constant1: tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5} constant2: 3.0 - } - - constant constant1asLarge { - type: tensor(x[3]) - file: constant1asLarge.json + constant1asLarge tensor(x[3]): file:constant1asLarge.json } function foo1() { diff --git a/model-integration/src/test/models/vespa/legacy_syntax.model b/model-integration/src/test/models/vespa/legacy_syntax.model new file mode 100644 index 00000000000..2a5031a5ff9 --- /dev/null +++ b/model-integration/src/test/models/vespa/legacy_syntax.model @@ -0,0 +1,26 @@ +model legacy_syntax { + + # Syntax not supported in rank profiles which probably should be removed on Vespa 9 + input1: tensor(name{}, x[3]) + input2: tensor(x[3]) + + constants { + constant1: tensor(x[3]):{{x:0}:0.5, {x:1}:1.5, {x:2}:2.5} + constant2: 3.0 + } + + # Syntax to be removed on Vespa 9 + constant constant1asLarge { + type: tensor(x[3]) + file: constant1asLarge.json + } + + function foo1() { + expression: file:test.expression + } + + function foo2() { + expression: reduce(sum(input1 * input2, name) * constant(constant1asLarge), max, x) * constant2 + } + +}
\ No newline at end of file diff --git a/node-admin/pom.xml b/node-admin/pom.xml index b5808ebc699..b58d20e8877 100644 --- a/node-admin/pom.xml +++ b/node-admin/pom.xml @@ -56,25 +56,25 @@ <!-- Compile --> <dependency> + <groupId>com.google.code.findbugs</groupId> + <artifactId>jsr305</artifactId> + </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> <artifactId>orchestrator-restapi</artifactId> <version>${project.version}</version> - <scope>compile</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>org.apache.velocity</groupId> <artifactId>velocity-engine-core</artifactId> - <scope>compile</scope> <exclusions> <exclusion> <!-- Must use the one provided by Jdisc to prevent two instances of slf4j classes. --> diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java index 30fb9ac03de..b820bf42287 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/StorageMaintainer.java @@ -153,7 +153,7 @@ public class StorageMaintainer { rules.add(CoredumpCleanupRule.forContainer(context.paths().underVespaHome("var/crash"))); if (context.node().membership().map(m -> m.type().hasContainer()).orElse(false)) - rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/vespa/qrs")).list(), + rules.add(new LinearCleanupRule(() -> FileFinder.files(context.paths().underVespaHome("logs/vespa/access")).list(), fa -> monthNormalizer.apply(fa.lastModifiedTime()), Priority.LOWEST, Priority.HIGHEST)); if (context.nodeType() == NodeType.tenant && context.node().membership().map(m -> m.type().isAdmin()).orElse(false)) diff --git a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java index 0d0b8bd335e..3004fb6490b 100644 --- a/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java +++ b/node-admin/src/main/java/com/yahoo/vespa/hosted/node/admin/maintenance/coredump/CoredumpHandler.java @@ -9,7 +9,6 @@ import com.yahoo.vespa.hosted.node.admin.nodeadmin.ConvergenceException; import com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentContext; import com.yahoo.vespa.hosted.node.admin.task.util.file.FileFinder; import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixPath; -import com.yahoo.vespa.hosted.node.admin.task.util.file.UnixUser; import com.yahoo.vespa.hosted.node.admin.task.util.fs.ContainerPath; import com.yahoo.vespa.hosted.node.admin.task.util.process.Terminal; @@ -88,14 +87,6 @@ public class CoredumpHandler { ContainerPath containerCrashPath = context.paths().of(crashPatchInContainer, context.users().vespa()); ContainerPath containerProcessingPath = containerCrashPath.resolve(PROCESSING_DIRECTORY_NAME); - // TODO (freva): Remove after 7.584 - UnixUser vespaUser = context.users().vespa(); - UnixPath processingPath = new UnixPath(containerProcessingPath); - processingPath.getAttributesIfExists().ifPresent(attr -> { - if (attr.ownerId() != vespaUser.uid()) processingPath.setOwnerId(vespaUser.uid()); - if (attr.groupId() != vespaUser.gid()) processingPath.setGroupId(vespaUser.gid()); - }); - updateMetrics(context, containerCrashPath); if (throwIfCoreBeingWritten) { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java index 3f07a8f5c90..98a85085b4a 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/configserver/noderepository/RealNodeRepositoryTest.java @@ -135,7 +135,7 @@ public class RealNodeRepositoryTest { hostname, new NodeAttributes() .withRestartGeneration(1) - .withDockerImage(DockerImage.fromString("registry.example.com/image-1:6.2.3"))); + .withDockerImage(DockerImage.fromString("registry.example.com/repo/image-1:6.2.3"))); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java index 12482b07b91..7db8cb79949 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/container/image/ContainerImageDownloaderTest.java @@ -21,7 +21,7 @@ public class ContainerImageDownloaderTest { ContainerEngineMock podman = new ContainerEngineMock().asyncImageDownload(true); ContainerImageDownloader downloader = new ContainerImageDownloader(podman); TaskContext context = new TestTaskContext(); - DockerImage image = DockerImage.fromString("registry.example.com/vespa:7.42"); + DockerImage image = DockerImage.fromString("registry.example.com/repo/vespa:7.42"); assertFalse("Download started", downloader.get(context, image, RegistryCredentials.none)); assertFalse("Download pending", downloader.get(context, image, RegistryCredentials.none)); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java index 5649da7c4ea..c22f2b0ab50 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/ContainerFailTest.java @@ -23,7 +23,7 @@ public class ContainerFailTest { @Test public void test() { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage"); + DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { ContainerName containerName = new ContainerName("host1"); String hostname = "host1.test.yahoo.com"; diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java index 79546d8cf78..eab320733ee 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/MultiContainerTest.java @@ -21,8 +21,8 @@ public class MultiContainerTest { @Test public void test() { - DockerImage image1 = DockerImage.fromString("registry.example.com/image1"); - DockerImage image2 = DockerImage.fromString("registry.example.com/image2"); + DockerImage image1 = DockerImage.fromString("registry.example.com/repo/image1"); + DockerImage image2 = DockerImage.fromString("registry.example.com/repo/image2"); try (ContainerTester tester = new ContainerTester(List.of(image1, image2))) { addAndWaitForNode(tester, "host1.test.yahoo.com", image1); NodeSpec nodeSpec2 = addAndWaitForNode(tester, "host2.test.yahoo.com", image2); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java index 277f9906dde..8498fa57084 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RebootTest.java @@ -23,7 +23,7 @@ import static org.mockito.ArgumentMatchers.eq; public class RebootTest { private final String hostname = "host1.test.yahoo.com"; - private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage"); + private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); @Test public void test() { diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java index 0675626c0e2..9a481959c63 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/integration/RestartTest.java @@ -22,7 +22,7 @@ public class RestartTest { @Test public void test() { - DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage:1.2.3"); + DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image:1.2.3"); try (ContainerTester tester = new ContainerTester(List.of(dockerImage))) { String hostname = "host1.test.yahoo.com"; NodeSpec nodeSpec = NodeSpec.Builder.testSpec(hostname).wantedDockerImage(dockerImage).build(); diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java index 1b5c51082c5..94c2ce6d106 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/maintenance/sync/SyncFileInfoTest.java @@ -25,12 +25,12 @@ public class SyncFileInfoTest { private static final FileSystem fileSystem = TestFileSystem.create(); private static final URI nodeArchiveUri = URI.create("s3://vespa-data-bucket/vespa/music/main/h432a/"); - private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/qrs/access.log.20210211"); - private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/access.log.20210212.zst"); - private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/qrs/access-json.log.20210213.zst"); - private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/qrs/JsonAccessLog.default.20210214.zst"); - private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210210"); - private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/qrs/ConnectionLog.default.20210212.zst"); + private static final Path accessLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210211"); + private static final Path accessLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/access.log.20210212.zst"); + private static final Path accessLogPath3 = fileSystem.getPath("/opt/vespa/logs/access/access-json.log.20210213.zst"); + private static final Path accessLogPath4 = fileSystem.getPath("/opt/vespa/logs/access/JsonAccessLog.20210214.zst"); + private static final Path connectionLogPath1 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210210"); + private static final Path connectionLogPath2 = fileSystem.getPath("/opt/vespa/logs/access/ConnectionLog.20210212.zst"); private static final Path vespaLogPath1 = fileSystem.getPath("/opt/vespa/logs/vespa.log"); private static final Path vespaLogPath2 = fileSystem.getPath("/opt/vespa/logs/vespa.log-2021-02-12"); private static final Path zkLogPath0 = fileSystem.getPath("/opt/vespa/logs/zookeeper.configserver.0.log"); @@ -47,17 +47,17 @@ public class SyncFileInfoTest { assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, true); assertForLogFile(accessLogPath3, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/access-json.log.20210213.zst", NONE, false); - assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.default.20210214.zst", NONE, true); - assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.default.20210214.zst", NONE, false); + assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, true); + assertForLogFile(accessLogPath4, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/access/JsonAccessLog.20210214.zst", NONE, false); } @Test public void connection_logs() { assertForLogFile(connectionLogPath1, null, null, true); - assertForLogFile(connectionLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.default.20210210.zst", ZSTD, false); + assertForLogFile(connectionLogPath1, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210210.zst", ZSTD, false); - assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.default.20210212.zst", NONE, true); - assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.default.20210212.zst", NONE, false); + assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, true); + assertForLogFile(connectionLogPath2, "s3://vespa-data-bucket/vespa/music/main/h432a/logs/connection/ConnectionLog.20210212.zst", NONE, false); } @Test diff --git a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java index 2eff2c64cec..d31c341532f 100644 --- a/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java +++ b/node-admin/src/test/java/com/yahoo/vespa/hosted/node/admin/nodeagent/NodeAgentImplTest.java @@ -64,7 +64,7 @@ public class NodeAgentImplTest { private static final String hostName = "host1.test.yahoo.com"; private final NodeAgentContextSupplier contextSupplier = mock(NodeAgentContextSupplier.class); - private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/dockerImage"); + private final DockerImage dockerImage = DockerImage.fromString("registry.example.com/repo/image"); private final ContainerOperations containerOperations = mock(ContainerOperations.class); private final NodeRepository nodeRepository = mock(NodeRepository.class); private final Orchestrator orchestrator = mock(Orchestrator.class); @@ -193,7 +193,7 @@ public class NodeAgentImplTest { @Test public void containerIsNotStoppedIfNewImageMustBePulled() { - final DockerImage newDockerImage = DockerImage.fromString("registry.example.com/new-image"); + final DockerImage newDockerImage = DockerImage.fromString("registry.example.com/repo/new-image"); final NodeSpec node = nodeBuilder(NodeState.active) .wantedDockerImage(newDockerImage).currentDockerImage(dockerImage) .wantedVespaVersion(vespaVersion).currentVespaVersion(vespaVersion) diff --git a/node-repository/CMakeLists.txt b/node-repository/CMakeLists.txt index fe4e09dc8a1..b4482f0daa7 100644 --- a/node-repository/CMakeLists.txt +++ b/node-repository/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(node-repository) +install_jar(node-repository-jar-with-dependencies.jar) install(FILES src/main/config/node-repository.xml DESTINATION conf/configserver-app) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java index e6408a0345b..4476befe70a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/Node.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; @@ -55,6 +56,7 @@ public final class Node implements Nodelike { private final Optional<ClusterSpec.Type> exclusiveToClusterType; private final Optional<String> switchHostname; private final List<TrustStoreItem> trustStoreItems; + private final Optional<CloudAccount> cloudAccount; /** Record of the last event of each type happening to this node */ private final History history; @@ -84,7 +86,8 @@ public final class Node implements Nodelike { Flavor flavor, Status status, State state, Optional<Allocation> allocation, History history, NodeType type, Reports reports, Optional<String> modelName, Optional<TenantName> reservedTo, Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, - Optional<String> switchHostname, List<TrustStoreItem> trustStoreItems) { + Optional<String> switchHostname, List<TrustStoreItem> trustStoreItems, + Optional<CloudAccount> cloudAccount) { this.id = Objects.requireNonNull(id, "A node must have an ID"); this.hostname = requireNonEmptyString(hostname, "A node must have a hostname"); this.ipConfig = Objects.requireNonNull(ipConfig, "A node must a have an IP config"); @@ -101,7 +104,8 @@ public final class Node implements Nodelike { this.exclusiveToApplicationId = Objects.requireNonNull(exclusiveToApplicationId, "exclusiveToApplicationId cannot be null"); this.exclusiveToClusterType = Objects.requireNonNull(exclusiveToClusterType, "exclusiveToClusterType cannot be null"); this.switchHostname = requireNonEmptyString(switchHostname, "switchHostname cannot be null"); - this.trustStoreItems = trustStoreItems.stream().distinct().collect(Collectors.toUnmodifiableList()); + this.trustStoreItems = Objects.requireNonNull(trustStoreItems).stream().distinct().collect(Collectors.toUnmodifiableList()); + this.cloudAccount = Objects.requireNonNull(cloudAccount); if (state == State.active) requireNonEmpty(ipConfig.primary(), "Active node " + hostname + " must have at least one valid IP address"); @@ -115,6 +119,11 @@ public final class Node implements Nodelike { if (!ipConfig.pool().ipSet().isEmpty()) throw new IllegalArgumentException("A child node cannot have an IP address pool"); if (modelName.isPresent()) throw new IllegalArgumentException("A child node cannot have model name set"); if (switchHostname.isPresent()) throw new IllegalArgumentException("A child node cannot have switch hostname set"); + if (cloudAccount().isPresent()) throw new IllegalArgumentException("A child node cannot have cloud account set"); + } + + if (cloudAccount.isPresent() && exclusiveToApplicationId.isEmpty()) { + throw new IllegalArgumentException("Host in a custom cloud account must be exclusive to an application"); } if (type != NodeType.host && reservedTo.isPresent()) @@ -218,6 +227,11 @@ public final class Node implements Nodelike { return trustStoreItems; } + /** Returns the cloud account of this host. This is empty if the host is in the zone's default account */ + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + /** * Returns a copy of this where wantToFail is set to true and history is updated to reflect this. */ @@ -306,13 +320,15 @@ public final class Node implements Nodelike { /** Returns a node with the status assigned to the given value */ public Node with(Status status) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, + trustStoreItems, cloudAccount); } /** Returns a node with the type assigned to the given value */ public Node with(NodeType type) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, history, type, - reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, + trustStoreItems, cloudAccount); } /** Returns a node with the flavor assigned to the given value */ @@ -320,31 +336,37 @@ public final class Node implements Nodelike { if (flavor.equals(this.flavor)) return this; History updateHistory = history.with(new History.Event(History.Event.Type.resized, agent, instant)); return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, allocation, updateHistory, type, - reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, + trustStoreItems, cloudAccount); } /** Returns a copy of this with the reboot generation set to generation */ public Node withReboot(Generation generation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status.withReboot(generation), state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } /** Returns a copy of this with given id set */ public Node withId(String id) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } /** Returns a copy of this with model name set to given value */ public Node withModelName(String modelName) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, Optional.of(modelName), reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } /** Returns a copy of this with model name cleared */ public Node withoutModelName() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, Optional.empty(), reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, Optional.empty(), reservedTo, + exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems, + cloudAccount); } /** Returns a copy of this with a history record saying it was detected to be down at this instant */ @@ -378,55 +400,66 @@ public final class Node implements Nodelike { */ public Node with(Allocation allocation) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + Optional.of(allocation), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } /** Returns a new Node without an allocation. */ public Node withoutAllocation() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - Optional.empty(), history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + Optional.empty(), history, type, reports, modelName, reservedTo, + exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems, + cloudAccount); } /** Returns a copy of this node with IP config set to the given value. */ public Node with(IP.Config ipConfig) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } /** Returns a copy of this node with the parent hostname assigned to the given value. */ public Node withParentHostname(String parentHostname) { return new Node(id, ipConfig, hostname, Optional.of(parentHostname), flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } public Node withReservedTo(TenantName tenant) { if (type != NodeType.host) throw new IllegalArgumentException("Only host nodes can be reserved, " + hostname + " has type " + type); return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, Optional.of(tenant), exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } /** Returns a copy of this node which is not reserved to a tenant */ public Node withoutReservedTo() { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, Optional.empty(), exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, Optional.empty(), + exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems, + cloudAccount); } public Node withExclusiveToApplicationId(ApplicationId exclusiveTo) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, Optional.ofNullable(exclusiveTo), + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } public Node withExclusiveToClusterType(ClusterSpec.Type exclusiveTo) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + Optional.ofNullable(exclusiveTo), switchHostname, trustStoreItems, cloudAccount); } /** Returns a copy of this node with switch hostname set to given value */ public Node withSwitchHostname(String switchHostname) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, Optional.ofNullable(switchHostname), trustStoreItems, cloudAccount); } /** Returns a copy of this node with switch hostname unset */ @@ -459,6 +492,12 @@ public final class Node implements Nodelike { return this.with(newStatus).with(newHistory); } + /** Returns a copy of this node with wanted OS version set to given version */ + public Node withWantedOsVersion(Optional<Version> version) { + if (status.osVersion().wanted().equals(version)) return this; + return with(status.withOsVersion(status.osVersion().withWanted(version))); + } + /** Returns a copy of this node with firmware verified at the given instant */ public Node withFirmwareVerifiedAt(Instant instant) { var newStatus = status.withFirmwareVerifiedAt(instant); @@ -469,18 +508,20 @@ public final class Node implements Nodelike { /** Returns a copy of this node with the given history. */ public Node with(History history) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } public Node with(Reports reports) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } public Node with(List<TrustStoreItem> trustStoreItems) { return new Node(id, ipConfig, hostname, parentHostname, flavor, status, state, - allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, exclusiveToClusterType, switchHostname, - trustStoreItems); + allocation, history, type, reports, modelName, reservedTo, exclusiveToApplicationId, + exclusiveToClusterType, switchHostname, trustStoreItems, cloudAccount); } private static Optional<String> requireNonEmptyString(Optional<String> value, String message) { @@ -602,6 +643,7 @@ public final class Node implements Nodelike { } public static class Builder { + private final String id; private final String hostname; private final Flavor flavor; @@ -620,6 +662,7 @@ public final class Node implements Nodelike { private Reports reports; private History history; private List<TrustStoreItem> trustStoreItems; + private CloudAccount cloudAccount; private Builder(String id, String hostname, Flavor flavor, State state, NodeType type) { this.id = id; @@ -694,14 +737,21 @@ public final class Node implements Nodelike { return this; } + public Builder cloudAccount(CloudAccount cloudAccount) { + this.cloudAccount = cloudAccount; + return this; + } + public Node build() { return new Node(id, Optional.ofNullable(ipConfig).orElse(IP.Config.EMPTY), hostname, Optional.ofNullable(parentHostname), flavor, Optional.ofNullable(status).orElseGet(Status::initial), state, Optional.ofNullable(allocation), Optional.ofNullable(history).orElseGet(History::empty), type, Optional.ofNullable(reports).orElseGet(Reports::new), Optional.ofNullable(modelName), Optional.ofNullable(reservedTo), Optional.ofNullable(exclusiveToApplicationId), Optional.ofNullable(exclusiveToClusterType), Optional.ofNullable(switchHostname), - Optional.ofNullable(trustStoreItems).orElseGet(List::of)); + Optional.ofNullable(trustStoreItems).orElseGet(List::of), + Optional.ofNullable(cloudAccount)); } + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java index 43a78e3a315..41fa9499353 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/AllocationOptimizer.java @@ -47,7 +47,7 @@ public class AllocationOptimizer { limits = Limits.of(new ClusterResources(minimumNodes, 1, NodeResources.unspecified()), new ClusterResources(maximumNodes, maximumNodes, NodeResources.unspecified())); else - limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec().type(), nodeRepository, clusterModel.application().id()); + limits = atLeast(minimumNodes, limits).fullySpecified(current.clusterSpec(), nodeRepository, clusterModel.application().id()); Optional<AllocatableClusterResources> bestAllocation = Optional.empty(); NodeList hosts = nodeRepository.nodes().list().hosts(); for (int groups = limits.min().groups(); groups <= limits.max().groups(); groups++) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java index ef45592ad18..36b32f0b099 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/Limits.java @@ -60,10 +60,10 @@ public class Limits { return resources; } - public Limits fullySpecified(ClusterSpec.Type type, NodeRepository nodeRepository, ApplicationId applicationId) { + public Limits fullySpecified(ClusterSpec clusterSpec, NodeRepository nodeRepository, ApplicationId applicationId) { if (this.isEmpty()) throw new IllegalStateException("Unspecified limits can not be made fully specified"); - var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(type, applicationId); + var defaultResources = new CapacityPolicies(nodeRepository).defaultNodeResources(clusterSpec, applicationId, clusterSpec.isExclusive()); var specifiedMin = min.nodeResources().isUnspecified() ? min.with(defaultResources) : min; var specifiedMax = max.nodeResources().isUnspecified() ? max.with(defaultResources) : max; return new Limits(specifiedMin, specifiedMax); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java index 1548038e19d..edffa817f64 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerInstance.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSortedSet; +import com.yahoo.config.provision.CloudAccount; import java.util.Objects; import java.util.Optional; @@ -21,14 +22,16 @@ public class LoadBalancerInstance { private final Set<Integer> ports; private final Set<String> networks; private final Set<Real> reals; + private final Optional<CloudAccount> cloudAccount; public LoadBalancerInstance(DomainName hostname, Optional<DnsZone> dnsZone, Set<Integer> ports, Set<String> networks, - Set<Real> reals) { + Set<Real> reals, Optional<CloudAccount> cloudAccount) { this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.ports = ImmutableSortedSet.copyOf(requirePorts(ports)); this.networks = ImmutableSortedSet.copyOf(Objects.requireNonNull(networks, "networks must be non-null")); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals, "targets must be non-null")); + this.cloudAccount = Objects.requireNonNull(cloudAccount, "cloudAccount must be non-null"); } /** Fully-qualified domain name of this load balancer. This hostname can be used for query and feed */ @@ -56,9 +59,14 @@ public class LoadBalancerInstance { return reals; } + /** Cloud account of this load balancer. This is empty if the load balancer is in the zone's default account */ + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + /** Returns a copy of this with reals set to given reals */ public LoadBalancerInstance withReals(Set<Real> reals) { - return new LoadBalancerInstance(hostname, dnsZone, ports, networks, reals); + return new LoadBalancerInstance(hostname, dnsZone, ports, networks, reals, cloudAccount); } private static Set<Integer> requirePorts(Set<Integer> ports) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java index ef127e04fba..0e388dac9dd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerService.java @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.lb; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; @@ -22,8 +21,8 @@ public interface LoadBalancerService { */ LoadBalancerInstance create(LoadBalancerSpec spec, boolean force); - /** Permanently remove load balancer for given application cluster */ - void remove(ApplicationId application, ClusterSpec.Id cluster); + /** Permanently remove given load balancer */ + void remove(LoadBalancer loadBalancer); /** Returns the protocol supported by this load balancer service */ Protocol protocol(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java index 7da28bff930..df92a6ee44d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerServiceMock.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSet; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; @@ -60,14 +59,15 @@ public class LoadBalancerServiceMock implements LoadBalancerService { Optional.of(new DnsZone("zone-id-1")), Collections.singleton(4443), ImmutableSet.of("10.2.3.0/24", "10.4.5.0/24"), - spec.reals()); + spec.reals(), + spec.cloudAccount()); instances.put(id, instance); return instance; } @Override - public void remove(ApplicationId application, ClusterSpec.Id cluster) { - instances.remove(new LoadBalancerId(application, cluster)); + public void remove(LoadBalancer loadBalancer) { + instances.remove(loadBalancer.id()); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java index f29cdeaa768..19739111415 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/LoadBalancerSpec.java @@ -3,9 +3,11 @@ package com.yahoo.vespa.hosted.provision.lb; import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import java.util.Objects; +import java.util.Optional; import java.util.Set; /** @@ -18,11 +20,14 @@ public class LoadBalancerSpec { private final ApplicationId application; private final ClusterSpec.Id cluster; private final Set<Real> reals; + private final Optional<CloudAccount> cloudAccount; - public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals) { + public LoadBalancerSpec(ApplicationId application, ClusterSpec.Id cluster, Set<Real> reals, + Optional<CloudAccount> cloudAccount) { this.application = Objects.requireNonNull(application); this.cluster = Objects.requireNonNull(cluster); this.reals = ImmutableSortedSet.copyOf(Objects.requireNonNull(reals)); + this.cloudAccount = Objects.requireNonNull(cloudAccount); } /** Owner of the load balancer */ @@ -40,4 +45,8 @@ public class LoadBalancerSpec { return reals; } + /** Cloud account to use when satisfying this */ + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java index fa9db9cd800..3d0cd3ef1e1 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerService.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.provision.lb; import ai.vespa.http.DomainName; -import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeType; @@ -11,11 +10,11 @@ import java.util.Optional; import java.util.Set; /** - * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exists by default in the shared + * This implementation of {@link LoadBalancerService} returns the load balancer(s) that exist by default in the shared * routing layer. * - * Since such load balancers always exist, we can return the hostname of the routing layer VIP and the networks of the - * proxy nodes directly. Nothing has to be provisioned. + * Since such load balancers always exist, we can return the hostname of the routing layer VIP directly. Nothing has to + * be provisioned. * * @author ogronnesby */ @@ -33,11 +32,12 @@ public class SharedLoadBalancerService implements LoadBalancerService { Optional.empty(), Set.of(4443), Set.of(), - spec.reals()); + spec.reals(), + spec.cloudAccount()); } @Override - public void remove(ApplicationId application, ClusterSpec.Id cluster) { + public void remove(LoadBalancer loadBalancer) { // Do nothing, we have no external state to modify } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java index 6eaee7b33de..0fa406ab0ef 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainer.java @@ -245,10 +245,11 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); List<Node> hosts = hostProvisioner.provisionHosts(provisionIndices, NodeType.host, nodeResources, - ApplicationId.defaultId(), osVersion, HostSharing.shared, Optional.empty()) - .stream() - .map(ProvisionedHost::generateHost) - .collect(Collectors.toList()); + ApplicationId.defaultId(), osVersion, HostSharing.shared, + Optional.empty(), Optional.empty()) + .stream() + .map(ProvisionedHost::generateHost) + .collect(Collectors.toList()); nodeRepository().nodes().addNodes(hosts, Agent.DynamicProvisioningMaintainer); return hosts; } catch (NodeAllocationException | IllegalArgumentException | IllegalStateException e) { @@ -293,7 +294,7 @@ public class DynamicProvisioningMaintainer extends NodeRepositoryMaintainer { // build() requires a version, even though it is not (should not be) used .vespaVersion(Vtag.currentVersion) .build(); - NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), nodeResources, false, true); + NodeSpec nodeSpec = NodeSpec.from(clusterCapacity.count(), nodeResources, false, true, Optional.empty()); int wantedGroups = 1; NodePrioritizer prioritizer = new NodePrioritizer(nodesAndHosts, applicationId, clusterSpec, nodeSpec, wantedGroups, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java index fbc3d236421..36ca58e6ccd 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/LoadBalancerExpirer.java @@ -80,7 +80,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { try { attempts.add(1); log.log(Level.INFO, () -> "Removing expired inactive " + lb.id()); - service.remove(lb.id().application(), lb.id().cluster()); + service.remove(lb); db.removeLoadBalancer(lb.id()); } catch (Exception e){ failed.add(lb.id()); @@ -113,7 +113,7 @@ public class LoadBalancerExpirer extends NodeRepositoryMaintainer { try { attempts.add(1); LOG.log(Level.INFO, () -> "Removing reals from inactive load balancer " + lb.id() + ": " + Sets.difference(lb.instance().get().reals(), reals)); - service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals), true); + service.create(new LoadBalancerSpec(lb.id().application(), lb.id().cluster(), reals, lb.instance().get().cloudAccount()), true); db.writeLoadBalancer(lb.with(lb.instance().map(instance -> instance.withReals(reals))), lb.state()); } catch (Exception e) { failed.add(lb.id()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java index 3aa09b1b667..8292b9590d7 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/node/Nodes.java @@ -628,10 +628,7 @@ public class Nodes { * @return the nodes in their new state */ public List<Node> upgradeOs(Predicate<Node> filter, Optional<Version> version) { - return performOn(filter, (node, lock) -> { - var newStatus = node.status().withOsVersion(node.status().osVersion().withWanted(version)); - return write(node.with(newStatus), lock); - }); + return performOn(filter, (node, lock) -> write(node.withWantedOsVersion(version), lock)); } /** Retire nodes matching given filter */ diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java index 13dd458c041..b4e304155a6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/CuratorDatabaseClient.java @@ -208,7 +208,8 @@ public class CuratorDatabaseClient { toState.isAllocated() ? node.allocation() : Optional.empty(), node.history().recordStateTransition(node.state(), toState, agent, clock.instant()), node.type(), node.reports(), node.modelName(), node.reservedTo(), - node.exclusiveToApplicationId(), node.exclusiveToClusterType(), node.switchHostname(), node.trustedCertificates()); + node.exclusiveToApplicationId(), node.exclusiveToClusterType(), node.switchHostname(), + node.trustedCertificates(), node.cloudAccount()); writeNode(toState, curatorTransaction, node, newNode); writtenNodes.add(newNode); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java index d953ab5b543..83180b2b136 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.persistence; import ai.vespa.http.DomainName; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -45,6 +46,7 @@ public class LoadBalancerSerializer { private static final String realsField = "reals"; private static final String ipAddressField = "ipAddress"; private static final String portField = "port"; + private static final String cloudAccountField = "cloudAccount"; public static byte[] toJson(LoadBalancer loadBalancer) { Slime slime = new Slime(); @@ -66,6 +68,7 @@ public class LoadBalancerSerializer { realObject.setString(ipAddressField, real.ipAddress()); realObject.setLong(portField, real.port()); })); + loadBalancer.instance().flatMap(LoadBalancerInstance::cloudAccount).ifPresent(cloudAccount -> root.setString(cloudAccountField, cloudAccount.value())); try { return SlimeUtils.toJsonBytes(slime); } catch (IOException e) { @@ -92,8 +95,9 @@ public class LoadBalancerSerializer { Optional<DomainName> hostname = optionalString(object.field(hostnameField), Function.identity()).filter(s -> !s.isEmpty()).map(DomainName::of); Optional<DnsZone> dnsZone = optionalString(object.field(dnsZoneField), DnsZone::new); + Optional<CloudAccount> cloudAccount = optionalString(object.field(cloudAccountField), CloudAccount::new); Optional<LoadBalancerInstance> instance = hostname.map(h -> new LoadBalancerInstance(h, dnsZone, ports, - networks, reals)); + networks, reals, cloudAccount)); return new LoadBalancer(LoadBalancerId.fromSerializedForm(object.field(idField).asString()), instance, diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index 7bbf5efba92..083b707b3c5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -9,6 +9,7 @@ import com.google.common.util.concurrent.UncheckedExecutionException; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.DockerImage; @@ -25,7 +26,6 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; -import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.node.Address; import com.yahoo.vespa.hosted.provision.node.Agent; @@ -99,6 +99,7 @@ public class NodeSerializer { private static final String exclusiveToClusterTypeKey = "exclusiveToClusterType"; private static final String switchHostnameKey = "switchHostname"; private static final String trustedCertificatesKey = "trustedCertificates"; + private static final String cloudAccountKey = "cloudAccount"; // Node resource fields private static final String flavorKey = "flavor"; @@ -192,6 +193,7 @@ public class NodeSerializer { node.exclusiveToApplicationId().ifPresent(applicationId -> object.setString(exclusiveToApplicationIdKey, applicationId.serializedForm())); node.exclusiveToClusterType().ifPresent(clusterType -> object.setString(exclusiveToClusterTypeKey, clusterType.name())); trustedCertificatesToSlime(node.trustedCertificates(), object.setArray(trustedCertificatesKey)); + node.cloudAccount().ifPresent(cloudAccount -> object.setString(cloudAccountKey, cloudAccount.value())); } private void toSlime(Flavor flavor, Cursor object) { @@ -275,7 +277,7 @@ public class NodeSerializer { ipAddressesFromSlime(object, ipAddressPoolKey), addressesFromSlime(object)), object.field(hostnameKey).asString(), - parentHostnameFromSlime(object), + SlimeUtils.optionalString(object.field(parentHostnameKey)), flavor, statusFromSlime(object), state, @@ -283,12 +285,13 @@ public class NodeSerializer { historyFromSlime(object), nodeTypeFromString(object.field(nodeTypeKey).asString()), Reports.fromSlime(object.field(reportsKey)), - modelNameFromSlime(object), - reservedToFromSlime(object.field(reservedToKey)), - exclusiveToApplicationIdFromSlime(object.field(exclusiveToApplicationIdKey)), - exclusiveToClusterTypeFromSlime(object.field(exclusiveToClusterTypeKey)), - switchHostnameFromSlime(object.field(switchHostnameKey)), - trustedCertificatesFromSlime(object)); + SlimeUtils.optionalString(object.field(modelNameKey)), + SlimeUtils.optionalString(object.field(reservedToKey)).map(TenantName::from), + SlimeUtils.optionalString(object.field(exclusiveToApplicationIdKey)).map(ApplicationId::fromSerializedForm), + SlimeUtils.optionalString(object.field(exclusiveToClusterTypeKey)).map(ClusterSpec.Type::from), + SlimeUtils.optionalString(object.field(switchHostnameKey)), + trustedCertificatesFromSlime(object), + SlimeUtils.optionalString(object.field(cloudAccountKey)).map(CloudAccount::new)); } private Status statusFromSlime(Inspector object) { @@ -303,12 +306,7 @@ public class NodeSerializer { object.field(wantToFailKey).asBool(), new OsVersion(versionFromSlime(object.field(osVersionKey)), versionFromSlime(object.field(wantedOsVersionKey))), - instantFromSlime(object.field(firmwareCheckKey))); - } - - private Optional<String> switchHostnameFromSlime(Inspector field) { - if (!field.valid()) return Optional.empty(); - return Optional.of(field.asString()); + SlimeUtils.optionalInstant(object.field(firmwareCheckKey))); } private Flavor flavorFromSlime(Inspector object) { @@ -373,35 +371,15 @@ public class NodeSerializer { private ClusterMembership clusterMembershipFromSlime(Inspector object) { return ClusterMembership.from(object.field(serviceIdKey).asString(), versionFromSlime(object.field(wantedVespaVersionKey)).get(), - containerImageRepoFromSlime(object.field(wantedContainerImageRepoKey))); + containerImageFromSlime(object.field(wantedContainerImageRepoKey))); } private Optional<Version> versionFromSlime(Inspector object) { - if ( ! object.valid()) return Optional.empty(); - return Optional.of(Version.fromString(object.asString())); - } - - private Optional<DockerImage> containerImageRepoFromSlime(Inspector object) { - if ( ! object.valid() || object.asString().isEmpty()) return Optional.empty(); - return Optional.of(DockerImage.fromString(object.asString())); + return object.valid() ? Optional.of(Version.fromString(object.asString())) : Optional.empty(); } private Optional<DockerImage> containerImageFromSlime(Inspector object) { - if ( ! object.valid()) return Optional.empty(); - return Optional.of(DockerImage.fromString(object.asString())); - } - - private Optional<Instant> instantFromSlime(Inspector object) { - if ( ! object.valid()) - return Optional.empty(); - return Optional.of(Instant.ofEpochMilli(object.asLong())); - } - - private Optional<String> parentHostnameFromSlime(Inspector object) { - if (object.field(parentHostnameKey).valid()) - return Optional.of(object.field(parentHostnameKey).asString()); - else - return Optional.empty(); + return SlimeUtils.optionalString(object).map(DockerImage::fromString); } private Set<String> ipAddressesFromSlime(Inspector object, String key) { @@ -412,43 +390,15 @@ public class NodeSerializer { private List<Address> addressesFromSlime(Inspector object) { return SlimeUtils.entriesStream(object.field(containersKey)) - .map(elem -> new Address(elem.field(containerHostnameKey).asString())) - .collect(Collectors.toList()); - } - - private Optional<String> modelNameFromSlime(Inspector object) { - if (object.field(modelNameKey).valid()) { - return Optional.of(object.field(modelNameKey).asString()); - } - return Optional.empty(); - } - - private Optional<TenantName> reservedToFromSlime(Inspector object) { - if (! object.valid()) return Optional.empty(); - if (object.type() != Type.STRING) - throw new IllegalArgumentException("Expected 'reservedTo' to be a string but is " + object); - return Optional.of(TenantName.from(object.asString())); - } - - private Optional<ApplicationId> exclusiveToApplicationIdFromSlime(Inspector object) { - if (! object.valid()) return Optional.empty(); - if (object.type() != Type.STRING) - throw new IllegalArgumentException("Expected 'exclusiveTo' to be a string but is " + object); - return Optional.of(ApplicationId.fromSerializedForm(object.asString())); - } - - private Optional<ClusterSpec.Type> exclusiveToClusterTypeFromSlime(Inspector object) { - if (! object.valid()) return Optional.empty(); - if (object.type() != Type.STRING) - throw new IllegalArgumentException("Expected 'exclusiveToClusterType' to be a string but is " + object); - return Optional.of(ClusterSpec.Type.from(object.asString())); + .map(elem -> new Address(elem.field(containerHostnameKey).asString())) + .collect(Collectors.toList()); } private List<TrustStoreItem> trustedCertificatesFromSlime(Inspector object) { return SlimeUtils.entriesStream(object.field(trustedCertificatesKey)) - .map(elem -> new TrustStoreItem(elem.field(fingerprintKey).asString(), - Instant.ofEpochMilli(elem.field(expiresKey).asLong()))) - .collect(Collectors.toList()); + .map(elem -> new TrustStoreItem(elem.field(fingerprintKey).asString(), + Instant.ofEpochMilli(elem.field(expiresKey).asLong()))) + .collect(Collectors.toList()); } // ----------------- Enum <-> string mappings ---------------------------------------- diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java index f7d5b966c12..12df01a7538 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/CapacityPolicies.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterResources; @@ -8,15 +9,18 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.Zone; -import com.yahoo.vespa.flags.FlagSource; +import com.yahoo.vespa.flags.JacksonFlag; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.flags.StringFlag; +import com.yahoo.vespa.flags.custom.SharedHost; import com.yahoo.vespa.hosted.provision.NodeRepository; -import java.util.function.Function; +import java.util.Map; +import java.util.TreeMap; import static com.yahoo.config.provision.NodeResources.Architecture; import static com.yahoo.vespa.flags.FetchVector.Dimension.APPLICATION_ID; -import static com.yahoo.vespa.flags.PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE; +import static java.util.Objects.requireNonNull; /** * Defines the policies for assigning cluster capacity in various environments @@ -27,24 +31,24 @@ import static com.yahoo.vespa.flags.PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTU public class CapacityPolicies { private final Zone zone; - private final Function<ClusterSpec.Type, Boolean> sharedHosts; - private final FlagSource flagSource; + private final JacksonFlag<SharedHost> sharedHosts; + private final StringFlag adminClusterNodeArchitecture; public CapacityPolicies(NodeRepository nodeRepository) { this.zone = nodeRepository.zone(); - this.sharedHosts = type -> PermanentFlags.SHARED_HOST.bindTo(nodeRepository.flagSource()).value().isEnabled(type.name()); - this.flagSource = nodeRepository.flagSource(); + this.sharedHosts = PermanentFlags.SHARED_HOST.bindTo(nodeRepository.flagSource()); + this.adminClusterNodeArchitecture = PermanentFlags.ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(nodeRepository.flagSource()); } - public Capacity applyOn(Capacity capacity, ApplicationId application) { - return capacity.withLimits(applyOn(capacity.minResources(), capacity, application), - applyOn(capacity.maxResources(), capacity, application)); + public Capacity applyOn(Capacity capacity, ApplicationId application, boolean exclusive) { + return capacity.withLimits(applyOn(capacity.minResources(), capacity, application, exclusive), + applyOn(capacity.maxResources(), capacity, application, exclusive)); } - private ClusterResources applyOn(ClusterResources resources, Capacity capacity, ApplicationId application) { + private ClusterResources applyOn(ClusterResources resources, Capacity capacity, ApplicationId application, boolean exclusive) { int nodes = decideSize(resources.nodes(), capacity.isRequired(), application.instance().isTester()); int groups = Math.min(resources.groups(), nodes); // cannot have more groups than nodes - var nodeResources = decideNodeResources(resources.nodeResources(), capacity.isRequired()); + var nodeResources = decideNodeResources(resources.nodeResources(), capacity.isRequired(), exclusive); return new ClusterResources(nodes, groups, nodeResources); } @@ -61,8 +65,8 @@ public class CapacityPolicies { } } - private NodeResources decideNodeResources(NodeResources target, boolean required) { - if (required) return target; + private NodeResources decideNodeResources(NodeResources target, boolean required, boolean exclusive) { + if (required || exclusive) return target; // Cannot downsize if resources are required, or exclusively allocated if (target.isUnspecified()) return target; // Cannot be modified // Dev does not cap the cpu or network of containers since usage is spotty: Allocate just a small amount exclusively @@ -76,21 +80,48 @@ public class CapacityPolicies { return target; } - public NodeResources defaultNodeResources(ClusterSpec.Type clusterType, ApplicationId applicationId) { - if (clusterType == ClusterSpec.Type.admin) { - Architecture architecture = Architecture.valueOf( - ADMIN_CLUSTER_NODE_ARCHITECTURE.bindTo(flagSource) - .with(APPLICATION_ID, applicationId.serializedForm()) - .value()); - - return zone.getCloud().dynamicProvisioning() && ! sharedHosts.apply(clusterType) ? - new NodeResources(0.5, 4, 50, 0.3).with(architecture) : - new NodeResources(0.5, 2, 50, 0.3).with(architecture); + public NodeResources defaultNodeResources(ClusterSpec clusterSpec, ApplicationId applicationId, boolean exclusive) { + if (clusterSpec.type() == ClusterSpec.Type.admin) { + Architecture architecture = architecture(applicationId); + + // The lowest amount resources that can be exclusive allocated (i.e. a matching host flavor for this exists) + NodeResources smallestExclusiveResources = new NodeResources(0.5, 4, 50, 0.3); + + if (clusterSpec.id().value().equals("cluster-controllers")) { + if (requiresExclusiveHost(clusterSpec.type(), exclusive)) { + return versioned(clusterSpec, Map.of(new Version("0"), smallestExclusiveResources)).with(architecture); + } + return versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(0.25, 1.14, 10, 0.3), + new Version("7.586.50"), new NodeResources(0.25, 1.333, 10, 0.3), + new Version("7.586.54"), new NodeResources(0.25, 1.14, 10, 0.3))) + .with(architecture); + } + + return (requiresExclusiveHost(clusterSpec.type(), exclusive) + ? versioned(clusterSpec, Map.of(new Version("0"), smallestExclusiveResources)) + : versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(0.5, 2, 50, 0.3)))) + .with(architecture); } - return zone.getCloud().dynamicProvisioning() ? - new NodeResources(2.0, 8, 50, 0.3) : - new NodeResources(1.5, 8, 50, 0.3); + return zone.getCloud().dynamicProvisioning() + ? versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(2.0, 8, 50, 0.3))) + : versioned(clusterSpec, Map.of(new Version("0"), new NodeResources(1.5, 8, 50, 0.3))); + } + + private Architecture architecture(ApplicationId instance) { + return Architecture.valueOf(adminClusterNodeArchitecture.with(APPLICATION_ID, instance.serializedForm()).value()); + } + + /** Returns whether an exclusive host is required for given cluster type and exclusivity requirement */ + private boolean requiresExclusiveHost(ClusterSpec.Type type, boolean exclusive) { + return zone.getCloud().dynamicProvisioning() && (exclusive || !sharedHosts.value().isEnabled(type.name())); + } + + /** Returns the resources for the newest version not newer than that requested in the cluster spec. */ + static NodeResources versioned(ClusterSpec spec, Map<Version, NodeResources> resources) { + return requireNonNull(new TreeMap<>(resources).floorEntry(spec.vespaVersion()), + "no default resources applicable for " + spec + " among: " + resources) + .getValue(); } /** @@ -98,6 +129,7 @@ public class CapacityPolicies { * A security feature which only makes sense for prod. */ public boolean decideExclusivity(Capacity capacity, boolean requestedExclusivity) { + if (zone.environment() == Environment.prod && capacity.cloudAccount().isPresent()) return true; // Implicit exclusive when using custom cloud account return requestedExclusivity && (capacity.isRequired() || zone.environment() == Environment.prod); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java index 763fe1cab6f..16ee8281b9a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/GroupPreparer.java @@ -107,7 +107,8 @@ public class GroupPreparer { application, osVersion, sharing, - Optional.of(cluster.type()))) + Optional.of(cluster.type()), + requestedNodes.cloudAccount())) .orElseGet(List::of); // At this point we have started provisioning of the hosts, the first priority is to make sure that diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index ce6ac9d5f5f..b849fccfaa5 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -41,6 +42,8 @@ public interface HostProvisioner { * @param osVersion the OS version to use. If this version does not exist, implementations may choose a suitable * fallback version. * @param sharing puts requirements on sharing or exclusivity of the host to be provisioned. + * @param clusterType provision host exclusively for this cluster type + * @param cloudAccount the cloud account to use * @return list of {@link ProvisionedHost} describing the provisioned nodes */ List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, @@ -49,7 +52,8 @@ public interface HostProvisioner { ApplicationId applicationId, Version osVersion, HostSharing sharing, - Optional<ClusterSpec.Type> clusterType); + Optional<ClusterSpec.Type> clusterType, + Optional<CloudAccount> cloudAccount); /** * Continue provisioning of given list of Nodes. diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java index f76968d67fd..a62edec52e4 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisioner.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.ApplicationTransaction; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; @@ -86,7 +87,7 @@ public class LoadBalancerProvisioner { ClusterSpec.Id clusterId = effectiveId(cluster); LoadBalancerId loadBalancerId = requireNonClashing(new LoadBalancerId(application, clusterId)); NodeList nodes = nodesOf(clusterId, application); - prepare(loadBalancerId, nodes); + prepare(loadBalancerId, nodes, requestedNodes.cloudAccount()); } } @@ -177,10 +178,10 @@ public class LoadBalancerProvisioner { return loadBalancerId; } - private void prepare(LoadBalancerId id, NodeList nodes) { + private void prepare(LoadBalancerId id, NodeList nodes, Optional<CloudAccount> cloudAccount) { Instant now = nodeRepository.clock().instant(); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); - Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, cloudAccount); LoadBalancer newLoadBalancer; LoadBalancer.State fromState = null; if (loadBalancer.isEmpty()) { @@ -199,8 +200,9 @@ public class LoadBalancerProvisioner { LoadBalancerId id = new LoadBalancerId(transaction.application(), cluster); Optional<LoadBalancer> loadBalancer = db.readLoadBalancer(id); if (loadBalancer.isEmpty()) throw new IllegalArgumentException("Could not active load balancer that was never prepared: " + id); + if (loadBalancer.get().instance().isEmpty()) throw new IllegalArgumentException("Activating " + id + ", but prepare never provisioned a load balancer instance"); - Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer); + Optional<LoadBalancerInstance> instance = provisionInstance(id, nodes, loadBalancer, loadBalancer.get().instance().get().cloudAccount()); LoadBalancer.State state = instance.isPresent() ? LoadBalancer.State.active : loadBalancer.get().state(); LoadBalancer newLoadBalancer = loadBalancer.get().with(instance).with(state, now); db.writeLoadBalancers(List.of(newLoadBalancer), loadBalancer.get().state(), transaction.nested()); @@ -208,7 +210,9 @@ public class LoadBalancerProvisioner { } /** Provision or reconfigure a load balancer instance, if necessary */ - private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes, Optional<LoadBalancer> currentLoadBalancer) { + private Optional<LoadBalancerInstance> provisionInstance(LoadBalancerId id, NodeList nodes, + Optional<LoadBalancer> currentLoadBalancer, + Optional<CloudAccount> cloudAccount) { boolean shouldDeactivateRouting = deactivateRouting.with(FetchVector.Dimension.APPLICATION_ID, id.application().serializedForm()) .value(); @@ -221,7 +225,7 @@ public class LoadBalancerProvisioner { if (hasReals(currentLoadBalancer, reals)) return currentLoadBalancer.get().instance(); log.log(Level.INFO, () -> "Provisioning instance for " + id + ", targeting: " + reals); try { - return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals), + return Optional.of(service.create(new LoadBalancerSpec(id.application(), id.cluster(), reals, cloudAccount), shouldDeactivateRouting || allowEmptyReals(currentLoadBalancer))); } catch (Exception e) { log.log(Level.WARNING, e, () -> "Could not (re)configure " + id + ", targeting: " + diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java index 768fa51f029..a26df62be27 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeAllocation.java @@ -193,7 +193,7 @@ class NodeAllocation { // In dynamic provisioned zones, exclusivity is violated if... if (nodeRepository.zone().getCloud().dynamicProvisioning()) { - // If either the parent is dedicated to a cluster type different from this cluster + // If either the parent is dedicated to a cluster type different from this cluster return ! candidate.parent.flatMap(Node::exclusiveToClusterType).map(cluster.type()::equals).orElse(true) || // or this cluster is requiring exclusivity, but the host is exclusive to a different owner (requestedNodes.isExclusive() && !candidate.parent.flatMap(Node::exclusiveToApplicationId).map(application::equals).orElse(false)); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 014c27b2ef1..64865c15529 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -17,9 +17,7 @@ import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; import com.yahoo.transaction.Mutex; -import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -99,27 +97,27 @@ public class NodeRepositoryProvisioner implements Provisioner { NodeResources resources; NodeSpec nodeSpec; if (requested.type() == NodeType.tenant) { - var actual = capacityPolicies.applyOn(requested, application); + boolean exclusive = capacityPolicies.decideExclusivity(requested, cluster.isExclusive()); + Capacity actual = capacityPolicies.applyOn(requested, application, exclusive); ClusterResources target = decideTargetResources(application, cluster, actual); - boolean exclusive = capacityPolicies.decideExclusivity(actual, cluster.isExclusive()); ensureRedundancy(target.nodes(), cluster, actual.canFail(), application); logIfDownscaled(requested.minResources().nodes(), actual.minResources().nodes(), cluster, logger); groups = target.groups(); - resources = getNodeResources(cluster, target.nodeResources(), application); - nodeSpec = NodeSpec.from(target.nodes(), resources, exclusive, actual.canFail()); + resources = getNodeResources(cluster, target.nodeResources(), application, exclusive); + nodeSpec = NodeSpec.from(target.nodes(), resources, exclusive, actual.canFail(), requested.cloudAccount()); } else { groups = 1; // type request with multiple groups is not supported - resources = getNodeResources(cluster, requested.minResources().nodeResources(), application); + resources = getNodeResources(cluster, requested.minResources().nodeResources(), application, true); nodeSpec = NodeSpec.from(requested.type()); } return asSortedHosts(preparer.prepare(application, cluster, nodeSpec, groups), resources); } - private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId) { + private NodeResources getNodeResources(ClusterSpec cluster, NodeResources nodeResources, ApplicationId applicationId, boolean exclusive) { return nodeResources.isUnspecified() - ? capacityPolicies.defaultNodeResources(cluster.type(), applicationId) + ? capacityPolicies.defaultNodeResources(cluster, applicationId, exclusive) : nodeResources; } @@ -180,7 +178,8 @@ public class NodeRepositoryProvisioner implements Provisioner { private ClusterResources initialResourcesFrom(Capacity requested, ClusterSpec clusterSpec, ApplicationId applicationId) { var initial = requested.minResources(); if (initial.nodeResources().isUnspecified()) - initial = initial.with(capacityPolicies.defaultNodeResources(clusterSpec.type(), applicationId)); + initial = initial.with(capacityPolicies.defaultNodeResources(clusterSpec, applicationId, + capacityPolicies.decideExclusivity(requested, clusterSpec.isExclusive()))); return initial; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java index b6c392c84db..90cd4a440f2 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeSpec.java @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.provision.provisioning; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeFlavors; @@ -68,6 +69,9 @@ public interface NodeSpec { /** Returns true if nodes with non-active parent hosts should be rejected */ boolean rejectNonActiveParent(); + /** Returns the cloud account to use when fulfilling this spec or empty if none is explicitly requested */ + Optional<CloudAccount> cloudAccount(); + /** * Returns true if a node with given current resources and current spare host resources can be resized * in-place to resources in this spec. @@ -77,8 +81,8 @@ public interface NodeSpec { return false; } - static NodeSpec from(int nodeCount, NodeResources resources, boolean exclusive, boolean canFail) { - return new CountNodeSpec(nodeCount, resources, exclusive, canFail); + static NodeSpec from(int nodeCount, NodeResources resources, boolean exclusive, boolean canFail, Optional<CloudAccount> cloudAccount) { + return new CountNodeSpec(nodeCount, resources, exclusive, canFail, cloudAccount); } static NodeSpec from(NodeType type) { @@ -92,12 +96,17 @@ public interface NodeSpec { private final NodeResources requestedNodeResources; private final boolean exclusive; private final boolean canFail; + private final Optional<CloudAccount> cloudAccount; - private CountNodeSpec(int count, NodeResources resources, boolean exclusive, boolean canFail) { + private CountNodeSpec(int count, NodeResources resources, boolean exclusive, boolean canFail, Optional<CloudAccount> cloudAccount) { this.count = count; this.requestedNodeResources = Objects.requireNonNull(resources, "Resources must be specified"); this.exclusive = exclusive; this.canFail = canFail; + this.cloudAccount = Objects.requireNonNull(cloudAccount); + if (cloudAccount.isPresent() && !exclusive) { + throw new IllegalArgumentException("Node spec with custom cloud account requires exclusive=true"); + } } @Override @@ -144,7 +153,7 @@ public interface NodeSpec { @Override public NodeSpec fraction(int divisor) { - return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail); + return new CountNodeSpec(count/divisor, requestedNodeResources, exclusive, canFail, cloudAccount); } @Override @@ -175,6 +184,11 @@ public interface NodeSpec { } @Override + public Optional<CloudAccount> cloudAccount() { + return cloudAccount; + } + + @Override public String toString() { return "request for " + count + " nodes with " + requestedNodeResources; } } @@ -243,6 +257,11 @@ public interface NodeSpec { } @Override + public Optional<CloudAccount> cloudAccount() { + return Optional.empty(); // Type spec does not support custom cloud accounts + } + + @Override public String toString() { return "request for all nodes of type '" + type + "'"; } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index dbc92b1dbdf..ef6c0da9169 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -76,8 +76,8 @@ class Preparer { if (requestedNodes.rejectNonActiveParent()) { NodeList activeHosts = allNodesAndHosts.nodes().state(Node.State.active).parents().nodeType(requestedNodes.type().hostType()); accepted = accepted.stream() - .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) - .collect(Collectors.toList()); + .filter(node -> node.parentHostname().isEmpty() || activeHosts.parentOf(node).isPresent()) + .collect(Collectors.toList()); } replace(acceptedNodes, accepted); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java index b5fd8c8111f..65071ad848d 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisionedHost.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; @@ -34,10 +35,11 @@ public class ProvisionedHost { private final List<Address> nodeAddresses; private final NodeResources nodeResources; private final Version osVersion; + private final Optional<CloudAccount> cloudAccount; public ProvisionedHost(String id, String hostHostname, Flavor hostFlavor, NodeType hostType, Optional<ApplicationId> exclusiveToApplicationId, Optional<ClusterSpec.Type> exclusiveToClusterType, - List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion) { + List<Address> nodeAddresses, NodeResources nodeResources, Version osVersion, Optional<CloudAccount> cloudAccount) { this.id = Objects.requireNonNull(id, "Host id must be set"); this.hostHostname = Objects.requireNonNull(hostHostname, "Host hostname must be set"); this.hostFlavor = Objects.requireNonNull(hostFlavor, "Host flavor must be set"); @@ -47,6 +49,7 @@ public class ProvisionedHost { this.nodeAddresses = validateNodeAddresses(nodeAddresses); this.nodeResources = Objects.requireNonNull(nodeResources, "Node resources must be set"); this.osVersion = Objects.requireNonNull(osVersion, "OS version must be set"); + this.cloudAccount = Objects.requireNonNull(cloudAccount, "Cloud account must be set"); if (!hostType.isHost()) throw new IllegalArgumentException(hostType + " is not a host"); } @@ -60,11 +63,12 @@ public class ProvisionedHost { /** Generate {@link Node} instance representing the provisioned physical host */ public Node generateHost() { - Node.Builder builder = Node - .create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, hostType) - .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); + Node.Builder builder = Node.create(id, IP.Config.of(Set.of(), Set.of(), nodeAddresses), hostHostname, hostFlavor, + hostType) + .status(Status.initial().withOsVersion(OsVersion.EMPTY.withCurrent(Optional.of(osVersion)))); exclusiveToApplicationId.ifPresent(builder::exclusiveToApplicationId); exclusiveToClusterType.ifPresent(builder::exclusiveToClusterType); + cloudAccount.ifPresent(builder::cloudAccount); return builder.build(); } @@ -82,6 +86,7 @@ public class ProvisionedHost { public List<Address> nodeAddresses() { return nodeAddresses; } public NodeResources nodeResources() { return nodeResources; } public Version osVersion() { return osVersion; } + public Optional<CloudAccount> cloudAccount() { return cloudAccount; } public String nodeHostname() { return nodeAddresses.get(0).hostname(); } @@ -98,12 +103,13 @@ public class ProvisionedHost { exclusiveToClusterType.equals(that.exclusiveToClusterType) && nodeAddresses.equals(that.nodeAddresses) && nodeResources.equals(that.nodeResources) && - osVersion.equals(that.osVersion); + osVersion.equals(that.osVersion) && + cloudAccount.equals(that.cloudAccount); } @Override public int hashCode() { - return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeAddresses, nodeResources, osVersion); + return Objects.hash(id, hostHostname, hostFlavor, hostType, exclusiveToApplicationId, exclusiveToClusterType, nodeAddresses, nodeResources, osVersion, cloudAccount); } @Override @@ -118,6 +124,7 @@ public class ProvisionedHost { ", nodeAddresses=" + nodeAddresses + ", nodeResources=" + nodeResources + ", osVersion=" + osVersion + + ", cloudAccount=" + cloudAccount + '}'; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java index f5b0c791d0a..7686a9a4885 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/LoadBalancersResponse.java @@ -3,33 +3,30 @@ package com.yahoo.vespa.hosted.provision.restapi; import com.yahoo.config.provision.ApplicationId; import com.yahoo.container.jdisc.HttpRequest; -import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.restapi.SlimeJsonResponse; import com.yahoo.slime.Cursor; -import com.yahoo.slime.JsonFormat; -import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerInstance; import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; import com.yahoo.vespa.hosted.provision.node.filter.ApplicationFilter; -import java.io.IOException; -import java.io.OutputStream; import java.util.List; import java.util.Optional; /** * @author mpolden */ -public class LoadBalancersResponse extends HttpResponse { +public class LoadBalancersResponse extends SlimeJsonResponse { private final NodeRepository nodeRepository; private final HttpRequest request; public LoadBalancersResponse(HttpRequest request, NodeRepository nodeRepository) { - super(200); this.request = request; this.nodeRepository = nodeRepository; + Cursor root = slime.setObject(); + toSlime(loadBalancers(), root); } private Optional<ApplicationId> application() { @@ -48,16 +45,9 @@ public class LoadBalancersResponse extends HttpResponse { return loadBalancers.asList(); } - @Override - public String getContentType() { return "application/json"; } - - @Override - public void render(OutputStream stream) throws IOException { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - Cursor loadBalancerArray = root.setArray("loadBalancers"); - - loadBalancers().forEach(lb -> { + private void toSlime(List<LoadBalancer> loadBalancers, Cursor object) { + Cursor loadBalancerArray = object.setArray("loadBalancers"); + loadBalancers.forEach(lb -> { Cursor lbObject = loadBalancerArray.addObject(); lbObject.setString("id", lb.id().serializedForm()); lbObject.setString("state", lb.state().name()); @@ -84,9 +74,10 @@ public class LoadBalancersResponse extends HttpResponse { realObject.setLong("port", real.port()); }); }); + lb.instance() + .flatMap(LoadBalancerInstance::cloudAccount) + .ifPresent(cloudAccount -> lbObject.setString("cloudAccount", cloudAccount.value())); }); - - new JsonFormat(true).encode(stream, slime); } } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java index d8e1828b10c..15ba96caa70 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodePatcher.java @@ -154,6 +154,8 @@ public class NodePatcher { return node.with(node.status().withVespaVersion(Version.fromString(asString(value)))); case "currentOsVersion" : return node.withCurrentOsVersion(Version.fromString(asString(value)), clock.instant()); + case "wantedOsVersion": // Node repository manages this field internally. Setting this manually should only be used for debugging purposes + return node.withWantedOsVersion(value.type() == Type.NIX ? Optional.empty() : Optional.of(Version.fromString(value.asString()))); case "currentFirmwareCheck": return node.withFirmwareVerifiedAt(Instant.ofEpochMilli(asLong(value))); case "failCount" : diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java index 729ea97d627..3659166c9da 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/restapi/NodesResponse.java @@ -183,6 +183,7 @@ class NodesResponse extends SlimeJsonResponse { node.switchHostname().ifPresent(switchHostname -> object.setString("switchHostname", switchHostname)); nodeRepository.archiveUris().archiveUriFor(node).ifPresent(uri -> object.setString("archiveUri", uri)); trustedCertsToSlime(node.trustedCertificates(), object); + node.cloudAccount().ifPresent(cloudAccount -> object.setString("cloudAccount", cloudAccount.value())); } private void toSlime(ApplicationId id, Cursor object) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index 0a125c5fb95..c09376ff103 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.provision.testutils; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provision.NodeResources; @@ -56,7 +57,8 @@ public class MockHostProvisioner implements HostProvisioner { @Override public List<ProvisionedHost> provisionHosts(List<Integer> provisionIndices, NodeType hostType, NodeResources resources, ApplicationId applicationId, Version osVersion, HostSharing sharing, - Optional<ClusterSpec.Type> clusterType) { + Optional<ClusterSpec.Type> clusterType, + Optional<CloudAccount> cloudAccount) { Flavor hostFlavor = this.hostFlavor.orElseGet(() -> flavors.stream().filter(f -> compatible(f, resources)) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + resources))); @@ -67,11 +69,12 @@ public class MockHostProvisioner implements HostProvisioner { hostHostname, hostFlavor, hostType, - Optional.empty(), + sharing == HostSharing.exclusive ? Optional.of(applicationId) : Optional.empty(), Optional.empty(), createAddressesForHost(hostType, hostFlavor, index), resources, - osVersion)); + osVersion, + cloudAccount)); } provisionedHosts.addAll(hosts); return hosts; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java index 40e38d752ff..4140588d1c8 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTest.java @@ -239,7 +239,7 @@ public class AutoscalingTest { ClusterSpec cluster1 = tester.clusterSpec(ClusterSpec.Type.container, "cluster1"); NodeResources defaultResources = - new CapacityPolicies(tester.nodeRepository()).defaultNodeResources(cluster1.type(), application1); + new CapacityPolicies(tester.nodeRepository()).defaultNodeResources(cluster1, application1, false); // deploy tester.deploy(application1, cluster1, Capacity.from(min, max)); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java index 8586704a426..0c164328086 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingTester.java @@ -32,7 +32,6 @@ import java.time.Duration; import java.time.Instant; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.Set; import java.util.function.IntFunction; @@ -314,7 +313,7 @@ class AutoscalingTester { } public Autoscaler.Advice autoscale(ApplicationId applicationId, ClusterSpec cluster, Capacity capacity) { - capacity = capacityPolicies.applyOn(capacity, applicationId); + capacity = capacityPolicies.applyOn(capacity, applicationId, capacityPolicies.decideExclusivity(capacity, cluster.isExclusive())); Application application = nodeRepository().applications().get(applicationId).orElse(Application.empty(applicationId)) .withCluster(cluster.id(), false, capacity); try (Mutex lock = nodeRepository().nodes().lock(applicationId)) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java index d54b41bfcdd..825e46865fe 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/lb/SharedLoadBalancerServiceTest.java @@ -26,7 +26,7 @@ public class SharedLoadBalancerServiceTest { @Test public void test_create_lb() { - var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals), false); + var lb = loadBalancerService.create(new LoadBalancerSpec(applicationId, clusterId, reals, Optional.empty()), false); assertEquals(HostName.of("vip.example.com"), lb.hostname()); assertEquals(Optional.empty(), lb.dnsZone()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java index 64cee87d942..30d0f673fe1 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/DynamicProvisioningMaintainerTest.java @@ -5,10 +5,13 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.Cloud; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; +import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeFlavors; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; @@ -542,6 +545,35 @@ public class DynamicProvisioningMaintainerTest { assertEquals(nodesBefore, nodesAfter); } + @Test + public void custom_cloud_account() { + DynamicProvisioningTester tester = new DynamicProvisioningTester(Cloud.builder().dynamicProvisioning(true).build(), + new MockNameResolver().mockAnyLookup()); + ProvisioningTester provisioningTester = tester.provisioningTester; + ApplicationId applicationId = ApplicationId.from("t1", "a1", "i1"); + + // Deployment requests capacity in custom account + ClusterSpec spec = ProvisioningTester.contentClusterSpec(); + ClusterResources resources = new ClusterResources(2, 1, new NodeResources(16, 24, 100, 1)); + CloudAccount cloudAccount = new CloudAccount("012345678912"); + Capacity capacity = Capacity.from(resources, resources, false, true, Optional.of(cloudAccount)); + List<HostSpec> prepared = provisioningTester.prepare(applicationId, spec, capacity); + + // Hosts are provisioned in requested account + tester.maintainer.maintain(); + List<ProvisionedHost> newHosts = tester.hostProvisioner.provisionedHosts(); + assertEquals(2, newHosts.size()); + assertTrue(newHosts.stream().allMatch(host -> host.cloudAccount().get().equals(cloudAccount))); + for (var host : newHosts) { + provisioningTester.nodeRepository().nodes().setReady(host.hostHostname(), Agent.operator, getClass().getSimpleName()); + } + provisioningTester.prepareAndActivateInfraApplication(DynamicProvisioningTester.tenantHostApp, NodeType.host); + NodeList activeHosts = provisioningTester.nodeRepository().nodes().list(Node.State.active).nodeType(NodeType.host); + assertEquals(2, activeHosts.size()); + assertTrue(activeHosts.stream().allMatch(host -> host.cloudAccount().get().equals(cloudAccount))); + assertEquals(2, provisioningTester.activate(applicationId, prepared).size()); + } + private void assertCfghost3IsActive(DynamicProvisioningTester tester) { assertEquals(5, tester.nodeRepository.nodes().list(Node.State.active).size()); assertEquals(3, tester.nodeRepository.nodes().list(Node.State.active).nodeType(NodeType.confighost).size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java index 1eecb442dc0..ce8fc2e9d03 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/LoadBalancerSerializerTest.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.provision.persistence; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSet; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.provision.lb.DnsZone; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; @@ -40,7 +41,8 @@ public class LoadBalancerSerializerTest { 4080), new Real(DomainName.of("real-2"), "127.0.0.2", - 4080)))), + 4080)), + Optional.of(new CloudAccount("012345678912")))), LoadBalancer.State.active, now); @@ -53,6 +55,7 @@ public class LoadBalancerSerializerTest { assertEquals(loadBalancer.state(), serialized.state()); assertEquals(loadBalancer.changedAt().truncatedTo(MILLIS), serialized.changedAt()); assertEquals(loadBalancer.instance().get().reals(), serialized.instance().get().reals()); + assertEquals(loadBalancer.instance().get().cloudAccount(), serialized.instance().get().cloudAccount()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java index 67b9569d8bf..bc7d3104a2f 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializerTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; @@ -488,6 +489,17 @@ public class NodeSerializerTest { assertEquals(trustStoreItems, node.trustedCertificates()); } + @Test + public void cloud_account_serialization() { + CloudAccount account = new CloudAccount("012345678912"); + Node node = Node.create("id", "host1.example.com", nodeFlavors.getFlavorOrThrow("default"), State.provisioned, NodeType.host) + .cloudAccount(account) + .exclusiveToApplicationId(ApplicationId.defaultId()) + .build(); + node = nodeSerializer.fromJson(State.provisioned, nodeSerializer.toJson(node)); + assertEquals(account, node.cloudAccount().get()); + } + private byte[] createNodeJson(String hostname, String... ipAddress) { String ipAddressJsonPart = ""; if (ipAddress.length > 0) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java index 97b664b6ba4..436cf880f1c 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTest.java @@ -73,7 +73,7 @@ public class DynamicProvisioningTest { mockHostProvisioner(hostProvisioner, "large", 3, null); // Provision shared hosts prepareAndActivate(application1, clusterSpec("mycluster"), 4, 1, resources); verify(hostProvisioner).provisionHosts(List.of(100, 101, 102, 103), NodeType.host, resources, application1, - Version.emptyVersion, HostSharing.any, Optional.of(ClusterSpec.Type.content)); + Version.emptyVersion, HostSharing.any, Optional.of(ClusterSpec.Type.content), Optional.empty()); // Total of 8 nodes should now be in node-repo, 4 active hosts and 4 active nodes assertEquals(8, tester.nodeRepository().nodes().list().size()); @@ -97,7 +97,7 @@ public class DynamicProvisioningTest { mockHostProvisioner(hostProvisioner, "large", 3, application3); prepareAndActivate(application3, clusterSpec("mycluster", true), 4, 1, resources); verify(hostProvisioner).provisionHosts(List.of(104, 105, 106, 107), NodeType.host, resources, application3, - Version.emptyVersion, HostSharing.exclusive, Optional.of(ClusterSpec.Type.content)); + Version.emptyVersion, HostSharing.exclusive, Optional.of(ClusterSpec.Type.content), Optional.empty()); // Total of 20 nodes should now be in node-repo, 8 active hosts and 12 active nodes assertEquals(20, tester.nodeRepository().nodes().list().size()); @@ -477,7 +477,7 @@ public class DynamicProvisioningTest { return provisionedHost; }) .collect(Collectors.toList()); - }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any(), any(), any()); + }).when(hostProvisioner).provisionHosts(any(), any(), any(), any(), any(), any(), any(), any()); } } diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java index 08058ef7b04..15e07f92292 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/LoadBalancerProvisionerTest.java @@ -5,6 +5,7 @@ import ai.vespa.http.DomainName; import com.google.common.collect.Iterators; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Capacity; +import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; @@ -17,6 +18,7 @@ import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.lb.LoadBalancer; +import com.yahoo.vespa.hosted.provision.lb.LoadBalancerList; import com.yahoo.vespa.hosted.provision.lb.Real; import com.yahoo.vespa.hosted.provision.node.Agent; import com.yahoo.vespa.hosted.provision.node.IP; @@ -308,6 +310,17 @@ public class LoadBalancerProvisionerTest { assertReals(app1, container1, Node.State.active); } + @Test + public void load_balancer_with_custom_cloud_account() { + ClusterResources resources = new ClusterResources(3, 1, nodeResources); + CloudAccount cloudAccount = new CloudAccount("012345678912"); + Capacity capacity = Capacity.from(resources, resources, false, true, Optional.of(cloudAccount)); + tester.activate(app1, prepare(app1, capacity, clusterRequest(ClusterSpec.Type.container, ClusterSpec.Id.from("c1")))); + LoadBalancerList loadBalancers = tester.nodeRepository().loadBalancers().list(); + assertEquals(1, loadBalancers.size()); + assertEquals(cloudAccount, loadBalancers.first().get().instance().get().cloudAccount().get()); + } + private void assertReals(ApplicationId application, ClusterSpec.Id cluster, Node.State... states) { List<LoadBalancer> loadBalancers = tester.nodeRepository().loadBalancers().list(application).cluster(cluster).asList(); assertEquals(1, loadBalancers.size()); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java index 94822c85a03..34219a15caa 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTest.java @@ -7,6 +7,9 @@ import com.yahoo.config.provision.Capacity; import com.yahoo.config.provision.ClusterMembership; import com.yahoo.config.provision.ClusterResources; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.ClusterSpec.Group; +import com.yahoo.config.provision.ClusterSpec.Id; +import com.yahoo.config.provision.ClusterSpec.Type; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.Flavor; @@ -37,6 +40,7 @@ import java.util.Collection; import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Random; import java.util.Set; @@ -49,6 +53,7 @@ import static java.time.temporal.ChronoUnit.MILLIS; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -1020,6 +1025,22 @@ public class ProvisioningTest { tester.activate(application, state.allHosts); } + @Test + public void test_versioned_resources() { + ClusterSpec.Builder spec = ClusterSpec.specification(Type.container, Id.from("id")).group(Group.from(0)); + Map<Version, NodeResources> resources = Map.of(new Version("7"), new NodeResources(2, 2, 2, 2), + new Version("8"), new NodeResources(3, 3, 3, 3), + new Version("6"), new NodeResources(1, 1, 1, 1)); + + assertThrows(NullPointerException.class, + () -> CapacityPolicies.versioned(spec.vespaVersion("5.0").build(), resources)); + assertEquals(new NodeResources(1, 1, 1, 1), CapacityPolicies.versioned(spec.vespaVersion("6.0").build(), resources)); + assertEquals(new NodeResources(2, 2, 2, 2), CapacityPolicies.versioned(spec.vespaVersion("7.0").build(), resources)); + assertEquals(new NodeResources(2, 2, 2, 2), CapacityPolicies.versioned(spec.vespaVersion("7.1").build(), resources)); + assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("8.0").build(), resources)); + assertEquals(new NodeResources(3, 3, 3, 3), CapacityPolicies.versioned(spec.vespaVersion("9.0").build(), resources)); + } + private SystemState prepare(ApplicationId application, int container0Size, int container1Size, int content0Size, int content1Size, NodeResources flavor, ProvisioningTester tester) { return prepare(application, tester, container0Size, container1Size, content0Size, content1Size, flavor, "6.42"); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index af94bba2e70..d3730106d54 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -30,7 +30,6 @@ import com.yahoo.vespa.curator.Curator; import com.yahoo.vespa.curator.mock.MockCurator; import com.yahoo.vespa.curator.transaction.CuratorTransaction; import com.yahoo.vespa.flags.FlagSource; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -153,7 +152,7 @@ public class ProvisioningTester { public Node node(String hostname) { return nodeRepository.nodes().node(hostname).get(); } public int decideSize(Capacity capacity, ApplicationId application) { - return capacityPolicies.applyOn(capacity, application).minResources().nodes(); + return capacityPolicies.applyOn(capacity, application, false).minResources().nodes(); } public Node patchNode(Node node, UnaryOperator<Node> patcher) { diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java index 5ac79a1cc24..6b1853b3893 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/restapi/NodesV2ApiTest.java @@ -777,7 +777,7 @@ public class NodesV2ApiTest { // Other node type does not return wanted OS version Response r = tester.container().handleRequest(new Request("http://localhost:8080/nodes/v2/node/host1.yahoo.com")); - assertFalse("Response omits wantedOsVersions field", r.getBodyAsString().contains("wantedOsVersion")); + assertFalse("Response omits wantedOsVersion field", r.getBodyAsString().contains("wantedOsVersion")); // Node updates its node object after upgrading OS assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", @@ -786,6 +786,16 @@ public class NodesV2ApiTest { "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); assertFile(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com"), "docker-node1-os-upgrade-complete.json"); + // Override wantedOsVersion + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantedOsVersion\": \"7.5.3\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost1.yahoo.com", + Utf8.toBytes("{\"wantedOsVersion\": \"7.5.2\"}"), + Request.Method.PATCH), + "{\"message\":\"Updated dockerhost1.yahoo.com\"}"); + // Another node upgrades assertResponse(new Request("http://localhost:8080/nodes/v2/node/dockerhost2.yahoo.com", Utf8.toBytes("{\"currentOsVersion\": \"7.5.2\"}"), diff --git a/orchestrator/CMakeLists.txt b/orchestrator/CMakeLists.txt index 77e1238d9bc..22044a9dace 100644 --- a/orchestrator/CMakeLists.txt +++ b/orchestrator/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(orchestrator) +install_jar(orchestrator-jar-with-dependencies.jar) diff --git a/parent/pom.xml b/parent/pom.xml index acd1ed6f25d..732a66710c6 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -763,6 +763,11 @@ </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-client</artifactId> + <version>${jetty.version}</version> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-continuation</artifactId> <version>${jetty.version}</version> </dependency> @@ -927,11 +932,11 @@ <antlr4.version>4.9.3</antlr4.version> <apache.httpclient.version>4.5.13</apache.httpclient.version> <apache.httpcore.version>4.4.13</apache.httpcore.version> - <apache.httpclient5.version>5.1.2</apache.httpclient5.version> + <apache.httpclient5.version>5.1.2</apache.httpclient5.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <asm.version>9.2</asm.version> <!-- Athenz dependencies. Make sure these dependencies match those in Vespa's internal repositories --> - <athenz.version>1.10.14</athenz.version> + <athenz.version>1.10.14</athenz.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <jjwt.version>0.11.2</jjwt.version> <aws.sdk.version>1.11.974</aws.sdk.version> <!-- Athenz END --> @@ -969,7 +974,7 @@ <maven-site-plugin.version>3.9.1</maven-site-plugin.version> <maven-source-plugin.version>3.2.1</maven-source-plugin.version> <mockito.version>4.0.0</mockito.version> - <onnxruntime.version>1.8.0</onnxruntime.version> + <onnxruntime.version>1.8.0</onnxruntime.version> <!-- WARNING: sync cloud-tenant-base-dependencies-enforcer/pom.xml --> <prometheus.client.version>0.6.0</prometheus.client.version> <protobuf.version>3.19.2</protobuf.version> <spifly.version>1.3.3</spifly.version> diff --git a/persistence/CMakeLists.txt b/persistence/CMakeLists.txt index 48a1265f81d..d87f172bfb5 100644 --- a/persistence/CMakeLists.txt +++ b/persistence/CMakeLists.txt @@ -4,14 +4,8 @@ vespa_define_module( fastos vespalog vespalib - staging_vespalib - fnet document - persistencetypes - config_cloudconfig vdslib - metrics - configdefinitions LIBS src/vespa/persistence diff --git a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp index 060215c4521..953cfcf733f 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp +++ b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.cpp @@ -14,7 +14,9 @@ DummyBucketExecutor::DummyBucketExecutor(size_t numExecutors) : _executor(std::make_unique<vespalib::ThreadStackExecutor>(numExecutors, 0x10000)), _lock(), _cond(), - _inFlight() + _inFlight(), + _defer_tasks(false), + _deferred_tasks() { } @@ -24,6 +26,15 @@ DummyBucketExecutor::~DummyBucketExecutor() { void DummyBucketExecutor::execute(const Bucket & bucket, std::unique_ptr<BucketTask> task) { + if (!_defer_tasks) { + internal_execute_no_defer(bucket, std::move(task)); + } else { + _deferred_tasks.emplace_back(bucket, std::move(task)); + } +} + +void +DummyBucketExecutor::internal_execute_no_defer(const Bucket& bucket, std::unique_ptr<BucketTask> task) { auto failed = _executor->execute(makeLambdaTask([this, bucket, bucketTask=std::move(task)]() { { std::unique_lock guard(_lock); @@ -45,6 +56,44 @@ DummyBucketExecutor::execute(const Bucket & bucket, std::unique_ptr<BucketTask> } void +DummyBucketExecutor::defer_new_tasks() { + std::lock_guard guard(_lock); + _defer_tasks = true; +} + +void +DummyBucketExecutor::schedule_all_deferred_tasks() { + DeferredTasks to_run; + { + std::lock_guard guard(_lock); + assert(_defer_tasks); + _deferred_tasks.swap(to_run); + } + for (auto& bucket_and_task : to_run) { + internal_execute_no_defer(bucket_and_task.first, std::move(bucket_and_task.second)); + } +} + +size_t +DummyBucketExecutor::num_deferred_tasks() const noexcept { + std::lock_guard guard(_lock); + return _deferred_tasks.size(); +} + +void +DummyBucketExecutor::schedule_single_deferred_task() { + std::pair<Bucket, std::unique_ptr<BucketTask>> bucket_and_task; + { + std::lock_guard guard(_lock); + assert(_defer_tasks); + assert(!_deferred_tasks.empty()); + bucket_and_task = std::move(_deferred_tasks.front()); + _deferred_tasks.pop_front(); + } + internal_execute_no_defer(bucket_and_task.first, std::move(bucket_and_task.second)); +} + +void DummyBucketExecutor::sync() { _executor->sync(); } diff --git a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.h b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.h index 86b497437fe..3e1432f7f54 100644 --- a/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.h +++ b/persistence/src/vespa/persistence/dummyimpl/dummy_bucket_executor.h @@ -4,6 +4,7 @@ #include <vespa/persistence/spi/bucketexecutor.h> #include <vespa/vespalib/util/threadexecutor.h> +#include <deque> #include <mutex> #include <condition_variable> #include <unordered_set> @@ -19,11 +20,21 @@ public: ~DummyBucketExecutor() override; void execute(const Bucket & bucket, std::unique_ptr<BucketTask> task) override; void sync(); + void defer_new_tasks(); + [[nodiscard]] size_t num_deferred_tasks() const noexcept; + void schedule_single_deferred_task(); + void schedule_all_deferred_tasks(); private: + void internal_execute_no_defer(const Bucket & bucket, std::unique_ptr<BucketTask> task); + + using DeferredTasks = std::deque<std::pair<Bucket, std::unique_ptr<BucketTask>>>; + std::unique_ptr<vespalib::SyncableThreadExecutor> _executor; - std::mutex _lock; + mutable std::mutex _lock; std::condition_variable _cond; - std::unordered_set<document::Bucket, document::Bucket::hash> _inFlight; + std::unordered_set<document::Bucket, document::Bucket::hash> _inFlight; + bool _defer_tasks; + DeferredTasks _deferred_tasks; }; } diff --git a/persistence/src/vespa/persistence/spi/CMakeLists.txt b/persistence/src/vespa/persistence/spi/CMakeLists.txt index eda3ffb228f..e4bae1c7551 100644 --- a/persistence/src/vespa/persistence/spi/CMakeLists.txt +++ b/persistence/src/vespa/persistence/spi/CMakeLists.txt @@ -17,5 +17,6 @@ vespa_add_library(persistence_spi OBJECT result.cpp selection.cpp test.cpp + types.cpp DEPENDS ) diff --git a/persistence/src/vespa/persistence/spi/bucketinfo.h b/persistence/src/vespa/persistence/spi/bucketinfo.h index 5759ceb2518..e6f16f38285 100644 --- a/persistence/src/vespa/persistence/spi/bucketinfo.h +++ b/persistence/src/vespa/persistence/spi/bucketinfo.h @@ -6,7 +6,7 @@ #pragma once -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> namespace vespalib { class asciistream; } diff --git a/persistence/src/vespa/persistence/spi/docentry.h b/persistence/src/vespa/persistence/spi/docentry.h index 9ad06b41e90..11a7a313802 100644 --- a/persistence/src/vespa/persistence/spi/docentry.h +++ b/persistence/src/vespa/persistence/spi/docentry.h @@ -13,7 +13,7 @@ #pragma once -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> #include <vespa/document/base/globalid.h> namespace storage::spi { diff --git a/persistence/src/vespa/persistence/spi/selection.h b/persistence/src/vespa/persistence/spi/selection.h index d4c7df8972f..4f6d9f9a61f 100644 --- a/persistence/src/vespa/persistence/spi/selection.h +++ b/persistence/src/vespa/persistence/spi/selection.h @@ -9,7 +9,7 @@ #pragma once #include "documentselection.h" -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> namespace storage::spi { diff --git a/persistencetypes/src/persistence/spi/types.cpp b/persistence/src/vespa/persistence/spi/types.cpp index 260355213d9..260355213d9 100644 --- a/persistencetypes/src/persistence/spi/types.cpp +++ b/persistence/src/vespa/persistence/spi/types.cpp diff --git a/persistencetypes/src/persistence/spi/types.h b/persistence/src/vespa/persistence/spi/types.h index a75b977c14a..a75b977c14a 100644 --- a/persistencetypes/src/persistence/spi/types.h +++ b/persistence/src/vespa/persistence/spi/types.h diff --git a/persistencetypes/.gitignore b/persistencetypes/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/persistencetypes/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/persistencetypes/CMakeLists.txt b/persistencetypes/CMakeLists.txt deleted file mode 100644 index a41f8eb6661..00000000000 --- a/persistencetypes/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - vespalog - vespalib - staging_vespalib - fnet - config_cloudconfig - document - - LIBS - src/persistence - src/persistence/spi -) diff --git a/persistencetypes/OWNERS b/persistencetypes/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/persistencetypes/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/persistencetypes/src/.gitignore b/persistencetypes/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/persistencetypes/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/persistencetypes/src/persistence/.gitignore b/persistencetypes/src/persistence/.gitignore deleted file mode 100644 index 444f5c50077..00000000000 --- a/persistencetypes/src/persistence/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/.depend -/Makefile -/libpersistencetypes.so.5.1 diff --git a/persistencetypes/src/persistence/CMakeLists.txt b/persistencetypes/src/persistence/CMakeLists.txt deleted file mode 100644 index 2ea62581c4e..00000000000 --- a/persistencetypes/src/persistence/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(persistencetypes - SOURCES - $<TARGET_OBJECTS:persistencetypes_spi> - INSTALL lib64 - DEPENDS -) diff --git a/persistencetypes/src/persistence/spi/CMakeLists.txt b/persistencetypes/src/persistence/spi/CMakeLists.txt deleted file mode 100644 index 5a5f335ef40..00000000000 --- a/persistencetypes/src/persistence/spi/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(persistencetypes_spi OBJECT - SOURCES - types.cpp - DEPENDS -) @@ -121,6 +121,7 @@ <module>standalone-container</module> <module>statistics</module> <module>storage</module> + <module>streamingvisitors</module> <module>tenant-base</module> <module>tenant-cd-api</module> <module>tenant-cd-commons</module> @@ -143,7 +144,6 @@ <module>vespa_jersey2</module> <module>vespajlib</module> <module>vespalog</module> - <module>vsm</module> <module>yolean</module> <module>zkfacade</module> <module>zookeeper-client-common</module> diff --git a/predicate-search/CMakeLists.txt b/predicate-search/CMakeLists.txt index a180c3fcd3e..eecd4c336ff 100644 --- a/predicate-search/CMakeLists.txt +++ b/predicate-search/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(predicate-search) +install_jar(predicate-search-jar-with-dependencies.jar) diff --git a/searchcommon/.gitignore b/searchcommon/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/searchcommon/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/searchcommon/CMakeLists.txt b/searchcommon/CMakeLists.txt deleted file mode 100644 index e5260ecb69e..00000000000 --- a/searchcommon/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - vespalog - vespalib - config_cloudconfig - configdefinitions - vespaeval - - TESTS - src/tests/attribute/config - src/tests/schema - - LIBS - src/vespa/searchcommon - src/vespa/searchcommon/attribute - src/vespa/searchcommon/common - src/vespa/searchcommon/config -) diff --git a/searchcommon/OWNERS b/searchcommon/OWNERS deleted file mode 100644 index cf65e9c8229..00000000000 --- a/searchcommon/OWNERS +++ /dev/null @@ -1,3 +0,0 @@ -geirst -baldersheim -toregge diff --git a/searchcommon/src/.gitignore b/searchcommon/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/searchcommon/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/searchcommon/src/vespa/searchcommon/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/CMakeLists.txt deleted file mode 100644 index 4d0d900bdf2..00000000000 --- a/searchcommon/src/vespa/searchcommon/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcommon - SOURCES - $<TARGET_OBJECTS:searchcommon_searchcommon_common> - $<TARGET_OBJECTS:searchcommon_searchcommon_attribute> - INSTALL lib64 - DEPENDS -) diff --git a/searchcommon/src/vespa/searchcommon/config/CMakeLists.txt b/searchcommon/src/vespa/searchcommon/config/CMakeLists.txt deleted file mode 100644 index e97068a5662..00000000000 --- a/searchcommon/src/vespa/searchcommon/config/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcommon_config INTERFACE - SOURCES - INSTALL lib64 - DEPENDS -) diff --git a/searchcommon/testrun/.gitignore b/searchcommon/testrun/.gitignore deleted file mode 100644 index 8f0724a7dba..00000000000 --- a/searchcommon/testrun/.gitignore +++ /dev/null @@ -1,12 +0,0 @@ -/test-report.html -/test-report.html.bottom -/test-report.html.entry -/test-report.html.summary -/test-report.html.top -test.*.*.desc -test.*.*.file.* -test.*.*.files.html -test.*.*.log -tmp.* -/test.*.*.result -/Makefile diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 7fb98cb2514..0f7d90b1491 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -7,15 +7,11 @@ vespa_define_module( vespalib metrics config_cloudconfig - fastlib_fast - juniper configdefinitions document searchlib messagebus - documentapi persistence - searchcorespi searchsummary fileacquirer @@ -41,12 +37,14 @@ vespa_define_module( src/vespa/searchcore/proton/server src/vespa/searchcore/proton/summaryengine src/vespa/searchcore/proton/test + src/vespa/searchcorespi + src/vespa/searchcorespi/flush + src/vespa/searchcorespi/index APPS src/apps/proton src/apps/tests src/apps/verify_ranksetup - src/apps/vespa-dump-feed src/apps/vespa-feed-bm src/apps/vespa-gen-testdocs src/apps/vespa-proton-cmd @@ -78,6 +76,7 @@ vespa_define_module( src/tests/proton/common/hw_info_sampler src/tests/proton/common/operation_rate_tracker src/tests/proton/common/state_reporter_utils + src/tests/proton/common/timer src/tests/proton/docsummary src/tests/proton/document_iterator src/tests/proton/documentdb @@ -158,6 +157,8 @@ vespa_define_module( src/tests/proton/statusreport src/tests/proton/summaryengine src/tests/proton/verify_ranksetup + src/tests/index/disk_indexes + src/tests/index/index_disk_layout TEST_DEPENDS messagebus_messagebus-test diff --git a/searchcore/src/apps/proton/CMakeLists.txt b/searchcore/src/apps/proton/CMakeLists.txt index 90aefd3dde5..400eb0eacc3 100644 --- a/searchcore/src/apps/proton/CMakeLists.txt +++ b/searchcore/src/apps/proton/CMakeLists.txt @@ -24,5 +24,4 @@ vespa_add_executable(searchcore_proton_app searchcore_proton_metrics searchcore_fconfig storageserver_storageapp - searchlib_searchlib_uca ) diff --git a/searchcore/src/apps/proton/proton.cpp b/searchcore/src/apps/proton/proton.cpp index c823ee8a09f..386ceb4eeda 100644 --- a/searchcore/src/apps/proton/proton.cpp +++ b/searchcore/src/apps/proton/proton.cpp @@ -203,6 +203,21 @@ buildTransportConfig() { return fnet::TransportConfig(std::max(1u, std::min(4u, numProcs/8))); } +class Transport { +public: + Transport(const fnet::TransportConfig & config, FastOS_ThreadPool & threadPool) + : _transport(config) + { + _transport.Start(&threadPool); + } + ~Transport() { + _transport.ShutDown(true); + } + FNET_Transport & transport() { return _transport; } +private: + FNET_Transport _transport; +}; + } void @@ -268,10 +283,8 @@ App::main(int argc, char **argv) setupSignals(); setup_fadvise(); FastOS_ThreadPool threadPool(128_Ki); - FNET_Transport transport(buildTransportConfig()); - transport.Start(&threadPool); - startAndRun(threadPool, transport, argc, argv); - transport.ShutDown(true); + Transport transport(buildTransportConfig(), threadPool); + startAndRun(threadPool, transport.transport(), argc, argv); } catch (const vespalib::InvalidCommandLineArgumentsException &e) { LOG(warning, "Invalid commandline arguments: '%s'", e.what()); return 1; diff --git a/searchcore/src/apps/tests/CMakeLists.txt b/searchcore/src/apps/tests/CMakeLists.txt index 532c89d6ab7..933be1a6f89 100644 --- a/searchcore/src/apps/tests/CMakeLists.txt +++ b/searchcore/src/apps/tests/CMakeLists.txt @@ -22,5 +22,4 @@ vespa_add_executable(searchcore_persistenceconformance_test_app TEST searchcore_fconfig vdstestlib persistence_persistence_conformancetest - searchlib_searchlib_uca ) diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 6b23f596835..ec48d8ebc65 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -149,7 +149,7 @@ public: schema, std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), - std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make()), std::make_shared<const AllocConfig>(), "client", docTypeName.getName()); diff --git a/searchcore/src/apps/vespa-dump-feed/.gitignore b/searchcore/src/apps/vespa-dump-feed/.gitignore deleted file mode 100644 index ea7ed317d25..00000000000 --- a/searchcore/src/apps/vespa-dump-feed/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -/.depend -/Makefile -/vespa-dump-feed -searchcore_vespa-dump-feed_app diff --git a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt b/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt deleted file mode 100644 index 086b047c55a..00000000000 --- a/searchcore/src/apps/vespa-dump-feed/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(searchcore_vespa-dump-feed_app - SOURCES - vespa-dump-feed.cpp - DEPENDS -) diff --git a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp b/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp deleted file mode 100644 index 98574752929..00000000000 --- a/searchcore/src/apps/vespa-dump-feed/vespa-dump-feed.cpp +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include <vespa/config/print/fileconfigwriter.h> -#include <vespa/document/config/config-documenttypes.h> -#include <vespa/document/repo/documenttyperepo.h> -#include <vespa/document/fieldvalue/document.h> -#include <vespa/documentapi/documentapi.h> -#include <vespa/messagebus/destinationsession.h> -#include <vespa/messagebus/rpcmessagebus.h> -#include <vespa/messagebus/network/rpcnetworkparams.h> -#include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/util/signalhandler.h> -#include <vespa/vespalib/process/process.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/objects/nbostream.h> -#include <vespa/config/common/exceptions.h> -#include <vespa/config/helper/configgetter.hpp> -#include <vespa/vespalib/util/signalhandler.h> - -typedef vespalib::SignalHandler SIG; - -//----------------------------------------------------------------------------- - -class OutputFile -{ -private: - FILE *file; - -public: - OutputFile(const std::string &name) - : file(fopen(name.c_str(), "w")) {} - bool valid() const { return (file != 0); } - void write(const char *data, size_t length) { - size_t res = fwrite(data, 1, length, file); - assert(res == length); - (void) res; - } - ~OutputFile() { fclose(file); } -}; - -//----------------------------------------------------------------------------- - -class FeedHandler : public mbus::IMessageHandler -{ -private: - mbus::RPCMessageBus _mbus; - mbus::DestinationSession::UP _session; - OutputFile &_idx; - OutputFile &_dat; - size_t _numDocs; - - void handleDocumentPut(const document::Document::SP & doc); - virtual void handleMessage(mbus::Message::UP message) override; - -public: - FeedHandler(std::shared_ptr<const document::DocumentTypeRepo> repo, OutputFile &idx, OutputFile &dat); - std::string getRoute() { return _session->getConnectionSpec(); } - virtual ~FeedHandler(); -}; - -void -FeedHandler::handleDocumentPut(const document::Document::SP & doc) -{ - if (doc) { - vespalib::nbostream datStream(12345); - vespalib::nbostream idxStream(12); - doc->serialize(datStream); - idxStream << uint64_t(datStream.size()); - _dat.write(datStream.peek(), datStream.size()); - _idx.write(idxStream.peek(), idxStream.size()); - ++_numDocs; - } -} - -void -FeedHandler::handleMessage(mbus::Message::UP message) -{ - mbus::Reply::UP reply; - documentapi::DocumentMessage::UP msg((documentapi::DocumentMessage*)message.release()); - switch (msg->getType()) { - case documentapi::DocumentProtocol::MESSAGE_PUTDOCUMENT: - handleDocumentPut(((documentapi::PutDocumentMessage&)(*msg)).getDocumentSP()); - break; - default: - break; - } - reply = msg->createReply(); // use default reply for all messages - msg->swapState(*reply); - _session->reply(std::move(reply)); // handle all messages synchronously -} - -FeedHandler::FeedHandler(std::shared_ptr<const document::DocumentTypeRepo> repo, OutputFile &idx, OutputFile &dat) - : _mbus(mbus::MessageBusParams().addProtocol(std::make_shared<documentapi::DocumentProtocol>(repo)), - mbus::RPCNetworkParams()), - _session(_mbus.getMessageBus() - .createDestinationSession(mbus::DestinationSessionParams() - .setBroadcastName(false) - .setMessageHandler(*this) - .setName("dump-feed"))), - _idx(idx), - _dat(dat), - _numDocs() -{ -} - -FeedHandler::~FeedHandler() -{ - _session.reset(); - fprintf(stderr, "%zu document puts dumped to disk\n", _numDocs); -} - -//----------------------------------------------------------------------------- - -class App -{ -public: - int main(int argc, char **argv); -}; - -template <typename CFG> -bool writeConfig(std::unique_ptr<CFG> cfg, const std::string &dirName) { - if (cfg.get() == 0) { - return false; - } - std::string fileName = dirName + "/" + CFG::CONFIG_DEF_NAME + ".cfg"; - try { - config::FileConfigWriter w(fileName); - return w.write(*cfg); - } catch (config::ConfigWriteException & e) { - fprintf(stderr, "Unable to write config to disk: %s\n", e.what()); - } - return false; -} - -template <typename CFG> -std::unique_ptr<CFG> getConfig() { - std::unique_ptr<CFG> ret(config::ConfigGetter<CFG>::getConfig("client")); - if (ret.get() == 0) { - fprintf(stderr, "error: could not obtain config (%s)\n", CFG::CONFIG_DEF_NAME.c_str()); - } - return ret; -} - -std::shared_ptr<const document::DocumentTypeRepo> getRepo() { - typedef document::config::DocumenttypesConfig DCFG; - std::unique_ptr<DCFG> dcfg = getConfig<DCFG>(); - std::shared_ptr<const document::DocumentTypeRepo> ret; - if (dcfg.get() != 0) { - ret.reset(new document::DocumentTypeRepo(*dcfg)); - } - return ret; -} - -void setupSignals() { - SIG::PIPE.ignore(); -} - -int usage() { - fprintf(stderr, "Usage: vespa-dump-feed <input-feed> <output-directory>\n\n"); - fprintf(stderr, " Takes an XML vespa feed as input and dumps its contents as serialized documents.\n"); - fprintf(stderr, " In addition to the actual documents, an index file containing document sizes\n"); - fprintf(stderr, " and the appropriate config file(s) needed for deserialization are also stored.\n"); - fprintf(stderr, " This utility can be run anywhere vespa-feeder can be run with default config id.\n"); - return 1; -} - -int -App::main(int argc, char **argv) -{ - setupSignals(); - if (argc != 3) { - return usage(); - } - std::string feedFile = argv[1]; - std::string dirName = argv[2]; - fprintf(stderr, "input feed: %s\n", feedFile.c_str()); - fprintf(stderr, "output directory: %s\n", dirName.c_str()); - vespalib::mkdir(dirName); - typedef document::config::DocumenttypesConfig DCFG; - if (!writeConfig(getConfig<DCFG>(), dirName)) { - fprintf(stderr, "error: could not save config to disk\n"); - return 1; - } - std::shared_ptr<const document::DocumentTypeRepo> repo = getRepo(); - if (repo.get() == 0) { - fprintf(stderr, "error: could not create document type repo\n"); - return 1; - } - { - OutputFile idxFile(dirName + "/doc.idx"); - OutputFile datFile(dirName + "/doc.dat"); - if (!idxFile.valid() || !datFile.valid()) { - fprintf(stderr, "error: could not open output document files\n"); - return 1; - } - FeedHandler feedHooks(repo, idxFile, datFile); - std::string route = feedHooks.getRoute(); - fprintf(stderr, "route to self: %s\n", route.c_str()); - std::string feedCmd(vespalib::make_string("vespa-feeder --route \"%s\" %s", - route.c_str(), feedFile.c_str())); - fprintf(stderr, "running feed command: %s\n", feedCmd.c_str()); - vespalib::string feederOutput; - bool feedingOk = vespalib::Process::run(feedCmd, feederOutput); - if (!feedingOk) { - fprintf(stderr, "error: feed command failed\n"); - fprintf(stderr, "feed command output:\n-----\n%s\n-----\n", feederOutput.c_str()); - return 1; - } - } - return 0; -} - -//----------------------------------------------------------------------------- - -int main(int argc, char **argv) { - App app; - return app.main(argc, argv); -} diff --git a/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt b/searchcore/src/tests/index/disk_indexes/CMakeLists.txt index 81b6bb0e8a9..81b6bb0e8a9 100644 --- a/searchcorespi/src/tests/index/disk_indexes/CMakeLists.txt +++ b/searchcore/src/tests/index/disk_indexes/CMakeLists.txt diff --git a/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp b/searchcore/src/tests/index/disk_indexes/disk_indexes_test.cpp index d22ad499316..d22ad499316 100644 --- a/searchcorespi/src/tests/index/disk_indexes/disk_indexes_test.cpp +++ b/searchcore/src/tests/index/disk_indexes/disk_indexes_test.cpp diff --git a/searchcorespi/src/tests/index/index_disk_layout/CMakeLists.txt b/searchcore/src/tests/index/index_disk_layout/CMakeLists.txt index 4e82cf1b9d2..4e82cf1b9d2 100644 --- a/searchcorespi/src/tests/index/index_disk_layout/CMakeLists.txt +++ b/searchcore/src/tests/index/index_disk_layout/CMakeLists.txt diff --git a/searchcorespi/src/tests/index/index_disk_layout/index_disk_layout_test.cpp b/searchcore/src/tests/index/index_disk_layout/index_disk_layout_test.cpp index e35225b2745..e35225b2745 100644 --- a/searchcorespi/src/tests/index/index_disk_layout/index_disk_layout_test.cpp +++ b/searchcore/src/tests/index/index_disk_layout/index_disk_layout_test.cpp diff --git a/searchcore/src/tests/proton/common/timer/.gitignore b/searchcore/src/tests/proton/common/timer/.gitignore new file mode 100644 index 00000000000..f0f08dcd650 --- /dev/null +++ b/searchcore/src/tests/proton/common/timer/.gitignore @@ -0,0 +1 @@ +searchcore_common_timer_test_app diff --git a/searchcore/src/tests/proton/common/timer/CMakeLists.txt b/searchcore/src/tests/proton/common/timer/CMakeLists.txt new file mode 100644 index 00000000000..89b9ecc688a --- /dev/null +++ b/searchcore/src/tests/proton/common/timer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_common_timer_test_app TEST + SOURCES + timer_test.cpp + DEPENDS + searchcore_pcommon +) +vespa_add_test(NAME searchcore_common_timer_test_app COMMAND searchcore_common_timer_test_app) diff --git a/staging_vespalib/src/tests/timer/timer_test.cpp b/searchcore/src/tests/proton/common/timer/timer_test.cpp index 1f0ee81e4e6..9eea67623b6 100644 --- a/staging_vespalib/src/tests/timer/timer_test.cpp +++ b/searchcore/src/tests/proton/common/timer/timer_test.cpp @@ -1,12 +1,11 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/scheduledexecutor.h> +#include <vespa/searchcore/proton/common/scheduledexecutor.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/fnet/transport.h> #include <vespa/fastos/thread.h> -using namespace vespalib; using vespalib::Executor; typedef Executor::Task Task; @@ -28,7 +27,7 @@ TEST("testScheduling") { FastOS_ThreadPool threadPool(64_Ki); FNET_Transport transport; transport.Start(&threadPool); - ScheduledExecutor timer(transport); + proton::ScheduledExecutor timer(transport); timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch1), 100ms, 200ms); timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch2), 500ms, 500ms); EXPECT_TRUE(latch1.await(60s)); @@ -42,7 +41,7 @@ TEST("testReset") { FastOS_ThreadPool threadPool(64_Ki); FNET_Transport transport; transport.Start(&threadPool); - ScheduledExecutor timer(transport); + proton::ScheduledExecutor timer(transport); timer.scheduleAtFixedRate(std::make_unique<TestTask>(latch1), 2s, 3s); timer.reset(); EXPECT_TRUE(!latch1.await(3s)); diff --git a/searchcore/src/tests/proton/docsummary/CMakeLists.txt b/searchcore/src/tests/proton/docsummary/CMakeLists.txt index 4286bbea224..274c1e302da 100644 --- a/searchcore/src/tests/proton/docsummary/CMakeLists.txt +++ b/searchcore/src/tests/proton/docsummary/CMakeLists.txt @@ -22,7 +22,6 @@ vespa_add_executable(searchcore_docsummary_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchlib_searchlib_uca ) vespa_add_executable(searchcore_summaryfieldconverter_test_app SOURCES diff --git a/searchcore/src/tests/proton/documentdb/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/CMakeLists.txt index d36368a8ebd..0cb678992b9 100644 --- a/searchcore/src/tests/proton/documentdb/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/CMakeLists.txt @@ -20,7 +20,6 @@ vespa_add_executable(searchcore_documentdb_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchlib_searchlib_uca ) vespa_add_test(NAME searchcore_documentdb_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/documentdb_test.sh DEPENDS searchcore_documentdb_test_app) diff --git a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt index 6472134f6e1..f217c4da740 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/configurer/CMakeLists.txt @@ -18,6 +18,5 @@ vespa_add_executable(searchcore_configurer_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchlib_searchlib_uca ) vespa_add_test(NAME searchcore_configurer_test_app COMMAND searchcore_configurer_test_app) diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt index 2706d183988..bc783c2eb3d 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/CMakeLists.txt @@ -20,6 +20,5 @@ vespa_add_executable(searchcore_document_subdbs_test_app TEST searchcore_grouping searchcore_proton_metrics searchcore_fconfig - searchlib_searchlib_uca ) vespa_add_test(NAME searchcore_document_subdbs_test_app COMMAND searchcore_document_subdbs_test_app) diff --git a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp index 9bc374b8386..2db34e45140 100644 --- a/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentbucketmover/documentbucketmover_test.cpp @@ -294,7 +294,7 @@ TEST_F(ControllerFixture, require_that_we_move_buckets_in_several_steps) EXPECT_EQ(0, numPending()); EXPECT_EQ(7u, docsMoved().size()); EXPECT_EQ(3u, bucketsModified().size()); - EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]); + ASSERT_EQ(_ready.bucket(2), bucketsModified()[0]); EXPECT_EQ(_notReady.bucket(3), bucketsModified()[1]); EXPECT_EQ(_notReady.bucket(4), bucketsModified()[2]); } @@ -481,6 +481,36 @@ TEST_F(ControllerFixture, explicitly_active_not_ready_bucket_can_be_moved_to_rea EXPECT_EQ(_notReady.bucket(3), bucketsModified()[0]); } +TEST_F(ControllerFixture, bucket_change_notification_is_not_lost_with_concurrent_bucket_movers) +{ + addReady(_ready.bucket(1)); + _bmj->recompute(); // Bucket 1 should be (and is) ready, bucket 2 is ready (but should not be). + _bucketExecutor.defer_new_tasks(); // Don't execute immediately, we need to force multiple pending moves + masterExecute([this]() { + deactivateBucket(_ready.bucket(2)); + _bmj->scanAndMove(4, 3); + // New deactivation received from above prior to completion of scan. This can happen since + // moves are asynchronous and the distributor can send new (de-)activations before the old move is done. + // In our case, we've enforced that another move is already pending in the bucket executor. + deactivateBucket(_ready.bucket(2)); + _bmj->scanAndMove(4, 3); + }); + sync(); + ASSERT_EQ(_bucketExecutor.num_deferred_tasks(), 2u); + _bucketExecutor.schedule_single_deferred_task(); + sync(); + // We have to fake that moving a document marks it as not found in the source sub DB. + // This doesn't automatically happen when using mocks. The most important part is that + // we ensure that moving isn't erroneously tested as if it were idempotent. + for (const auto& move : docsMoved()) { + failRetrieveForLid(move.getPrevLid()); + } + _bucketExecutor.schedule_single_deferred_task(); + sync(); + EXPECT_TRUE(_bmj->done()); + ASSERT_EQ(1u, bucketsModified().size()); + EXPECT_EQ(_ready.bucket(2), bucketsModified()[0]); +} TEST_F(ControllerFixture, require_that_notifyCreateBucket_causes_bucket_to_be_reconsidered_by_job) { diff --git a/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp b/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp index 46a6419e924..bc02f460b4e 100644 --- a/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp +++ b/searchcore/src/tests/proton/documentdb/executor_threading_service/executor_threading_service_test.cpp @@ -9,7 +9,6 @@ using namespace proton; using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; -using SharedFieldWriterExecutor = ThreadingServiceConfig::SharedFieldWriterExecutor; VESPA_THREAD_STACK_TAG(my_field_writer_executor) @@ -27,16 +26,13 @@ public: ExecutorThreadingServiceTest() : _transport(1), field_writer_executor(SequencedTaskExecutor::create(my_field_writer_executor, 3, 200)), - service() - { - } - void setup(uint32_t indexing_threads, SharedFieldWriterExecutor shared_field_writer) { - service = std::make_unique<ExecutorThreadingService>(_transport.shared(), + service(std::make_unique<ExecutorThreadingService>(_transport.shared(), _transport.transport(), _transport.clock(), - field_writer_executor.get(), + *field_writer_executor, nullptr, - ThreadingServiceConfig::make(indexing_threads, shared_field_writer)); + ThreadingServiceConfig::make())) + { } SequencedTaskExecutor* index_inverter() { return to_concrete_type(service->indexFieldInverter()); @@ -59,36 +55,8 @@ assert_executor(SequencedTaskExecutor* exec, uint32_t exp_executors, uint32_t ex EXPECT_EQ(exp_task_limit, exec->first_executor()->getTaskLimit()); } -TEST_F(ExecutorThreadingServiceTest, no_shared_field_writer_executor) -{ - setup(4, SharedFieldWriterExecutor::NONE); - EXPECT_NE(index_inverter(), index_writer()); - EXPECT_NE(index_writer(), attribute_writer()); - assert_executor(index_inverter(), 4, 100); - assert_executor(index_writer(), 4, 100); - assert_executor(attribute_writer(), 4, 100); -} - -TEST_F(ExecutorThreadingServiceTest, shared_executor_for_index_field_writers) -{ - setup(4, SharedFieldWriterExecutor::INDEX); - EXPECT_EQ(index_inverter(), index_writer()); - EXPECT_NE(index_inverter(), attribute_writer()); - assert_executor(index_inverter(), 8, 100); - assert_executor(attribute_writer(), 4, 100); -} - -TEST_F(ExecutorThreadingServiceTest, shared_executor_for_index_and_attribute_field_writers) -{ - setup(4, SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE); - EXPECT_EQ(index_inverter(), index_writer()); - EXPECT_EQ(index_inverter(), attribute_writer()); - assert_executor(index_inverter(), 12, 100); -} - TEST_F(ExecutorThreadingServiceTest, shared_field_writer_specified_from_the_outside) { - setup(4, SharedFieldWriterExecutor::DOCUMENT_DB); EXPECT_EQ(field_writer(), index_inverter()); EXPECT_EQ(field_writer(), index_writer()); EXPECT_EQ(field_writer(), attribute_writer()); @@ -97,7 +65,6 @@ TEST_F(ExecutorThreadingServiceTest, shared_field_writer_specified_from_the_outs TEST_F(ExecutorThreadingServiceTest, tasks_limits_can_be_updated) { - setup(4, SharedFieldWriterExecutor::NONE); service->set_task_limits(5, 7, 11); EXPECT_EQ(5, service->master_task_limit()); EXPECT_EQ(7, service->index().getTaskLimit()); diff --git a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp index 57989688a4f..9c68d7d5974 100644 --- a/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp +++ b/searchcore/src/tests/proton/documentdb/lid_space_compaction/lid_space_common.cpp @@ -106,7 +106,7 @@ MyHandler::createMoveOperation(const search::DocumentMetaData &document, uint32_ assert(document.lid > moveToLid); _moveFromLid = document.lid; const auto & entry = _docs[document.lid]; - auto op = std::make_unique<MoveOperation>(entry.first.bucketId, entry.first.timestamp, entry.second, + auto op = std::make_unique<MoveOperation>(entry.first.bucketId, storage::spi::Timestamp(entry.first.timestamp), entry.second, DbDocumentId(document.lid), 0); op->setTargetLid(moveToLid); return op; diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index fc8bd474813..1cee63ecfcc 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -14,54 +14,32 @@ using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; struct Fixture { ProtonConfig cfg; - Fixture(uint32_t baseLineIndexingThreads = 2, uint32_t master_task_limit = 2000, int32_t task_limit = 500) - : cfg(makeConfig(baseLineIndexingThreads, master_task_limit, task_limit)) + Fixture(uint32_t master_task_limit = 2000, int32_t task_limit = 500) + : cfg(makeConfig(master_task_limit, task_limit)) { } - ProtonConfig makeConfig(uint32_t baseLineIndexingThreads, uint32_t master_task_limit, int32_t task_limit) { + ProtonConfig makeConfig(uint32_t master_task_limit, int32_t task_limit) { ProtonConfigBuilder builder; - builder.indexing.threads = baseLineIndexingThreads; builder.indexing.tasklimit = task_limit; builder.feeding.masterTaskLimit = master_task_limit; return builder; } - ThreadingServiceConfig make(uint32_t cpuCores) { - return ThreadingServiceConfig::make(cfg, 0.5, HwInfo::Cpu(cpuCores)); - } - void assertIndexingThreads(uint32_t expIndexingThreads, uint32_t cpuCores) { - EXPECT_EQUAL(expIndexingThreads, make(cpuCores).indexingThreads()); + ThreadingServiceConfig make() { + return ThreadingServiceConfig::make(cfg); } }; -TEST_F("require that indexing threads are set based on cpu cores and feeding concurrency", Fixture) -{ - TEST_DO(f.assertIndexingThreads(2, 1)); - TEST_DO(f.assertIndexingThreads(2, 4)); - TEST_DO(f.assertIndexingThreads(2, 8)); - TEST_DO(f.assertIndexingThreads(2, 12)); - TEST_DO(f.assertIndexingThreads(3, 13)); - TEST_DO(f.assertIndexingThreads(3, 18)); - TEST_DO(f.assertIndexingThreads(4, 19)); - TEST_DO(f.assertIndexingThreads(4, 24)); - TEST_DO(f.assertIndexingThreads(11, 64)); -} - -TEST_F("require that indexing threads is always >= 1", Fixture(0)) -{ - TEST_DO(f.assertIndexingThreads(1, 0)); -} - TEST_F("require that task limits are set", Fixture) { - auto tcfg = f.make(24); + auto tcfg = f.make(); EXPECT_EQUAL(2000u, tcfg.master_task_limit()); EXPECT_EQUAL(500u, tcfg.defaultTaskLimit()); EXPECT_TRUE(tcfg.is_task_limit_hard()); } -TEST_F("require that negative task limit makes it soft", Fixture(2, 3000, -700)) +TEST_F("require that negative task limit makes it soft", Fixture(3000, -700)) { - auto tcfg = f.make(24); + auto tcfg = f.make(); EXPECT_EQUAL(3000u, tcfg.master_task_limit()); EXPECT_EQUAL(700u, tcfg.defaultTaskLimit()); EXPECT_FALSE(tcfg.is_task_limit_hard()); @@ -69,23 +47,21 @@ TEST_F("require that negative task limit makes it soft", Fixture(2, 3000, -700)) namespace { -void assertConfig(uint32_t exp_indexing_threads, uint32_t exp_master_task_limit, - uint32_t exp_default_task_limit, const ThreadingServiceConfig& config) { - EXPECT_EQUAL(exp_indexing_threads, config.indexingThreads()); +void assertConfig(uint32_t exp_master_task_limit, uint32_t exp_default_task_limit, const ThreadingServiceConfig& config) { EXPECT_EQUAL(exp_master_task_limit, config.master_task_limit()); EXPECT_EQUAL(exp_default_task_limit, config.defaultTaskLimit()); } } -TEST_FF("require that config can be somewhat updated", Fixture(), Fixture(2, 3000, 1000)) +TEST_FF("require that config can be somewhat updated", Fixture(), Fixture(3000, 1000)) { - auto cfg1 = f1.make(1); - assertConfig(2u, 2000, 500u, cfg1); - const auto cfg2 = f2.make(13); - assertConfig(3u, 3000u, 1000u, cfg2); + auto cfg1 = f1.make(); + assertConfig(2000, 500u, cfg1); + const auto cfg2 = f2.make(); + assertConfig(3000u, 1000u, cfg2); cfg1.update(cfg2); - assertConfig(2u, 3000u, 1000u, cfg1); // Indexing threads not changed + assertConfig(3000u, 1000u, cfg1); } TEST_MAIN() diff --git a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp index 0414990d74e..73311c110a9 100644 --- a/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/documentmetastore_test.cpp @@ -102,7 +102,7 @@ createBucketDB() void assertPut(const BucketId &bucketId, - const Timestamp ×tamp, + uint64_t timestamp, uint32_t lid, const GlobalId &gid, DocumentMetaStore &dms) diff --git a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp index b0f1220c768..2d675e82db2 100644 --- a/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lid_allocator/lid_allocator_test.cpp @@ -83,7 +83,7 @@ protected: std::vector<uint32_t> get_active_lids() { std::vector<uint32_t> result; - auto active_lids = _allocator.getActiveLids(); + const auto &active_lids = _allocator.getActiveLids(); uint32_t lid = active_lids.getNextTrueBit(1); while (lid < active_lids.size()) { if (active_lids.testBit(lid)) { diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index a30408a9301..62e8a3e553a 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -67,11 +67,11 @@ public: return Result(); } - Result put(const GlobalId &, const BucketId &, const Timestamp &, uint32_t, DocId, uint64_t) override { + Result put(const GlobalId &, const BucketId &, Timestamp , uint32_t, DocId, uint64_t) override { return Result(); } - bool updateMetaData(DocId, const BucketId &, const Timestamp &) override { + bool updateMetaData(DocId, const BucketId &, Timestamp ) override { return true; } diff --git a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp index 039524a237c..5c4d7aca41d 100644 --- a/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp +++ b/searchcore/src/tests/proton/feedoperation/feedoperation_test.cpp @@ -14,7 +14,6 @@ #include <vespa/searchcore/proton/feedoperation/splitbucketoperation.h> #include <vespa/searchcore/proton/feedoperation/updateoperation.h> #include <vespa/searchlib/query/base.h> -#include <persistence/spi/types.h> #include <vespa/document/base/documentid.h> #include <vespa/document/datatype/datatype.h> #include <vespa/document/fieldvalue/document.h> @@ -42,7 +41,6 @@ using document::config_builder::DocumenttypesConfigBuilderHelper; using document::config_builder::Struct; using document::config_builder::Map; using search::DocumentIdT; -using storage::spi::Timestamp; using namespace proton; namespace { @@ -59,6 +57,8 @@ const vespalib::string type_name = "test"; const vespalib::string header_name = type_name + ".header"; const vespalib::string body_name = type_name + ".body"; +const DocumentOperation::Timestamp TS_10(10); + const document::DocumentId docId("id::test::1"); BucketId toBucket(const GlobalId &gid) @@ -85,7 +85,7 @@ uint32_t getDocIdSize(const DocumentId &doc_id) void assertDocumentOperation(DocumentOperation &op, BucketId expBucket, uint32_t expDocSize) { EXPECT_EQUAL(expBucket, op.getBucketId()); - EXPECT_EQUAL(10u, op.getTimestamp().getValue()); + EXPECT_EQUAL(10u, op.getTimestamp()); EXPECT_EQUAL(expDocSize, op.getSerializedDocSize()); EXPECT_EQUAL(1u, op.getSubDbId()); EXPECT_EQUAL(2u, op.getLid()); @@ -138,7 +138,7 @@ TEST("require that toString() on derived classes are meaningful") BucketId bucket_id1(42); BucketId bucket_id2(43); BucketId bucket_id3(44); - Timestamp timestamp(10); + DocumentOperation::Timestamp timestamp(10); Document::SP doc(new Document); DbDocumentId db_doc_id; uint32_t sub_db_id = 1; @@ -247,7 +247,7 @@ TEST_F("require that we can serialize and deserialize update operations", Fixtur BucketId bucket(toBucket(docId.getGlobalId())); auto upd(f.makeUpdate()); { - UpdateOperation op(bucket, Timestamp(10), upd); + UpdateOperation op(bucket, 10, upd); op.serialize(stream); } { @@ -255,7 +255,7 @@ TEST_F("require that we can serialize and deserialize update operations", Fixtur op.deserialize(stream, *f._repo); EXPECT_EQUAL(*upd, *op.getUpdate()); EXPECT_EQUAL(bucket, op.getBucketId()); - EXPECT_EQUAL(10u, op.getTimestamp().getValue()); + EXPECT_EQUAL(10u, op.getTimestamp()); } } @@ -267,7 +267,7 @@ TEST_F("require that we can serialize and deserialize put operations", Fixture) uint32_t expSerializedDocSize = getDocSize(*doc); EXPECT_NOT_EQUAL(0u, expSerializedDocSize); { - PutOperation op(bucket, Timestamp(10), doc); + PutOperation op(bucket, 10, doc); op.setDbDocumentId({1, 2}); op.setPrevDbDocumentId({3, 4}); EXPECT_EQUAL(0u, op.getSerializedDocSize()); @@ -290,7 +290,7 @@ TEST_F("require that we can serialize and deserialize move operations", Fixture) uint32_t expSerializedDocSize = getDocSize(*doc); EXPECT_NOT_EQUAL(0u, expSerializedDocSize); { - MoveOperation op(bucket, Timestamp(10), doc, {3, 4}, 1); + MoveOperation op(bucket, TS_10 , doc, {3, 4}, 1); op.setTargetLid(2); EXPECT_EQUAL(0u, op.getSerializedDocSize()); op.serialize(stream); @@ -311,7 +311,7 @@ TEST_F("require that we can serialize and deserialize remove operations", Fixtur uint32_t expSerializedDocSize = getDocIdSize(docId); EXPECT_NOT_EQUAL(0u, expSerializedDocSize); { - RemoveOperationWithDocId op(bucket, Timestamp(10), docId); + RemoveOperationWithDocId op(bucket, TS_10 , docId); op.setDbDocumentId({1, 2}); op.setPrevDbDocumentId({3, 4}); EXPECT_EQUAL(0u, op.getSerializedDocSize()); @@ -335,7 +335,7 @@ TEST_F("require that we can serialize and deserialize remove by gid operations", vespalib::string expDocType = "testdoc_type"; EXPECT_NOT_EQUAL(0u, expSerializedDocSize); { - RemoveOperationWithGid op(bucket, Timestamp(10), gid, expDocType); + RemoveOperationWithGid op(bucket, TS_10 , gid, expDocType); op.setPrevDbDocumentId({3, 4}); EXPECT_EQUAL(0u, op.getSerializedDocSize()); op.serialize(stream); @@ -347,7 +347,7 @@ TEST_F("require that we can serialize and deserialize remove by gid operations", EXPECT_EQUAL(gid, op.getGlobalId()); EXPECT_EQUAL(expDocType, op.getDocType()); EXPECT_EQUAL(bucket, op.getBucketId()); - EXPECT_EQUAL(10u, op.getTimestamp().getValue()); + EXPECT_EQUAL(10u, op.getTimestamp()); EXPECT_EQUAL(expSerializedDocSize, op.getSerializedDocSize()); EXPECT_FALSE( op.getValidDbdId()); EXPECT_EQUAL(3u, op.getPrevSubDbId()); diff --git a/searchcore/src/tests/proton/flushengine/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/CMakeLists.txt index b45c4fa411c..a47b3f2c93f 100644 --- a/searchcore/src/tests/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/tests/proton/flushengine/CMakeLists.txt @@ -5,6 +5,7 @@ vespa_add_executable(searchcore_flushengine_test_app TEST DEPENDS searchcore_flushengine searchcore_pcommon + searchcore_test ) vespa_add_test( NAME searchcore_flushengine_test_app diff --git a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt index 9f0c777a4d7..608f80e38aa 100644 --- a/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt +++ b/searchcore/src/tests/proton/flushengine/prepare_restart_flush_strategy/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_executable(searchcore_flushengine_prepare_restart_flush_strategy_test_ SOURCES prepare_restart_flush_strategy_test.cpp DEPENDS - searchcorespi searchcore_flushengine + searchcore_test ) vespa_add_test( NAME searchcore_flushengine_prepare_restart_flush_strategy_test_app diff --git a/searchcore/src/tests/proton/matching/CMakeLists.txt b/searchcore/src/tests/proton/matching/CMakeLists.txt index c42a11be8ea..b4737be7819 100644 --- a/searchcore/src/tests/proton/matching/CMakeLists.txt +++ b/searchcore/src/tests/proton/matching/CMakeLists.txt @@ -12,7 +12,6 @@ vespa_add_executable(searchcore_matching_test_app TEST searchcore_bucketdb searchcore_pcommon searchcore_grouping - searchlib_searchlib_uca searchlib_test ) vespa_add_test(NAME searchcore_matching_test_app COMMAND searchcore_matching_test_app) diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 187c0463da3..e3d0b37c0d6 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -1,42 +1,44 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> #include <vespa/document/base/globalid.h> -#include <initializer_list> +#include <vespa/eval/eval/simple_value.h> +#include <vespa/eval/eval/tensor_spec.h> +#include <vespa/eval/eval/value_codec.h> #include <vespa/searchcommon/attribute/iattributecontext.h> -#include <vespa/searchcore/proton/test/bucketfactory.h> +#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> #include <vespa/searchcore/proton/documentmetastore/documentmetastore.h> #include <vespa/searchcore/proton/matching/fakesearchcontext.h> #include <vespa/searchcore/proton/matching/i_constant_value_repo.h> +#include <vespa/searchcore/proton/matching/match_context.h> +#include <vespa/searchcore/proton/matching/match_params.h> +#include <vespa/searchcore/proton/matching/match_tools.h> #include <vespa/searchcore/proton/matching/matcher.h> #include <vespa/searchcore/proton/matching/querynodes.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/matching/viewresolver.h> -#include <vespa/searchcore/proton/bucketdb/bucket_db_owner.h> +#include <vespa/searchcore/proton/test/bucketfactory.h> #include <vespa/searchlib/aggregation/aggregation.h> #include <vespa/searchlib/aggregation/grouping.h> #include <vespa/searchlib/aggregation/perdocexpression.h> #include <vespa/searchlib/attribute/extendableattributes.h> #include <vespa/searchlib/common/featureset.h> -#include <vespa/searchlib/engine/docsumrequest.h> -#include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/engine/docsumreply.h> +#include <vespa/searchlib/engine/docsumrequest.h> #include <vespa/searchlib/engine/searchreply.h> -#include <vespa/searchlib/test/mock_attribute_context.h> -#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/engine/searchrequest.h> #include <vespa/searchlib/fef/indexproperties.h> +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/ranksetup.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/query/tree/querybuilder.h> #include <vespa/searchlib/query/tree/stackdumpcreator.h> #include <vespa/searchlib/queryeval/isourceselector.h> -#include <vespa/vespalib/util/simple_thread_bundle.h> -#include <vespa/searchcore/proton/matching/match_params.h> -#include <vespa/searchcore/proton/matching/match_tools.h> -#include <vespa/searchcore/proton/matching/match_context.h> -#include <vespa/eval/eval/simple_value.h> -#include <vespa/eval/eval/tensor_spec.h> -#include <vespa/eval/eval/value_codec.h> +#include <vespa/searchlib/test/mock_attribute_context.h> #include <vespa/vespalib/objects/nbostream.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/simple_thread_bundle.h> #include <vespa/vespalib/util/testclock.h> +#include <initializer_list> #include <vespa/log/log.h> LOG_SETUP("matching_test"); @@ -47,6 +49,7 @@ using namespace search::aggregation; using namespace search::attribute; using namespace search::engine; using namespace search::expression; +using namespace search::fef::indexproperties::matching; using namespace search::fef; using namespace search::grouping; using namespace search::index; @@ -55,13 +58,12 @@ using namespace search::queryeval; using namespace search; using search::attribute::test::MockAttributeContext; +using search::fef::indexproperties::hitcollector::HeapSize; using search::index::schema::DataType; using storage::spi::Timestamp; -using search::fef::indexproperties::hitcollector::HeapSize; - -using vespalib::nbostream; using vespalib::eval::SimpleValue; using vespalib::eval::TensorSpec; +using vespalib::nbostream; void inject_match_phase_limiting(Properties &setup, const vespalib::string &attribute, size_t max_hits, bool descending) { @@ -1102,4 +1104,49 @@ TEST("require that docsum matcher can extract matching elements from single attr EXPECT_EQUAL(list[1], 3u); } +struct GlobalFilterParamsFixture { + BlueprintFactory factory; + search::fef::test::IndexEnvironment index_env; + RankSetup rank_setup; + Properties rank_properties; + GlobalFilterParamsFixture(double lower_limit, double upper_limit) + : factory(), + index_env(), + rank_setup(factory, index_env), + rank_properties() + { + rank_setup.set_global_filter_lower_limit(lower_limit); + rank_setup.set_global_filter_upper_limit(upper_limit); + } + void set_query_properties(vespalib::stringref lower_limit, vespalib::stringref upper_limit) { + rank_properties.add(GlobalFilterLowerLimit::NAME, lower_limit); + rank_properties.add(GlobalFilterUpperLimit::NAME, upper_limit); + } + AttributeBlueprintParams extract(uint32_t active_docids = 9, uint32_t docid_limit = 10) { + return MatchToolsFactory::extract_global_filter_params(rank_setup, rank_properties, active_docids, docid_limit); + } +}; + +TEST_F("global filter params are extracted from rank profile", GlobalFilterParamsFixture(0.2, 0.8)) +{ + auto params = f.extract(); + EXPECT_EQUAL(0.2, params.global_filter_lower_limit); + EXPECT_EQUAL(0.8, params.global_filter_upper_limit); +} + +TEST_F("global filter params are extracted from query", GlobalFilterParamsFixture(0.2, 0.8)) +{ + f.set_query_properties("0.15", "0.75"); + auto params = f.extract(); + EXPECT_EQUAL(0.15, params.global_filter_lower_limit); + EXPECT_EQUAL(0.75, params.global_filter_upper_limit); +} + +TEST_F("global filter params are scaled with active hit ratio", GlobalFilterParamsFixture(0.2, 0.8)) +{ + auto params = f.extract(5, 10); + EXPECT_EQUAL(0.12, params.global_filter_lower_limit); + EXPECT_EQUAL(0.48, params.global_filter_upper_limit); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt index 4a5dd2acb9d..aad11581e2f 100644 --- a/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt +++ b/searchcore/src/tests/proton/metrics/documentdb_job_trackers/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchcore_documentdb_job_trackers_test_app TEST documentdb_job_trackers_test.cpp DEPENDS searchcore_proton_metrics + searchcore_test ) vespa_add_test(NAME searchcore_documentdb_job_trackers_test_app COMMAND searchcore_documentdb_job_trackers_test_app) diff --git a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt index 6ab08602f67..4467bea4ab1 100644 --- a/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt +++ b/searchcore/src/tests/proton/metrics/job_tracked_flush/CMakeLists.txt @@ -4,5 +4,6 @@ vespa_add_executable(searchcore_job_tracked_flush_test_app TEST job_tracked_flush_test.cpp DEPENDS searchcore_proton_metrics + searchcore_test ) vespa_add_test(NAME searchcore_job_tracked_flush_test_app COMMAND searchcore_job_tracked_flush_test_app) diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 85f8e8171a8..8a12219de3c 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -106,7 +106,7 @@ struct DBConfigFixture { buildSchema(), std::make_shared<DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), - std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1)), + std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make()), std::make_shared<const AllocConfig>(), configId, docTypeName); diff --git a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt index 80c5521c763..207883aab0e 100644 --- a/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt +++ b/searchcore/src/tests/proton/server/memoryflush/CMakeLists.txt @@ -5,5 +5,6 @@ vespa_add_executable(searchcore_memoryflush_test_app TEST DEPENDS searchcore_server searchcore_flushengine + searchcore_test ) vespa_add_test(NAME searchcore_memoryflush_test_app COMMAND searchcore_memoryflush_test_app) diff --git a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp index a15bd8c67c9..2027ad56768 100644 --- a/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp +++ b/searchcore/src/tests/proton/server/shared_threading_service/shared_threading_service_test.cpp @@ -17,7 +17,7 @@ using vespalib::ISequencedTaskExecutor; using vespalib::SequencedTaskExecutor; ProtonConfig -make_proton_config(double concurrency) +make_proton_config(double concurrency, uint32_t indexing_threads = 1) { ProtonConfigBuilder builder; // This setup requires a minimum of 4 shared threads. @@ -26,8 +26,8 @@ make_proton_config(double concurrency) builder.flush.maxconcurrent = 1; builder.feeding.concurrency = concurrency; - builder.feeding.sharedFieldWriterExecutor = ProtonConfig::Feeding::SharedFieldWriterExecutor::DOCUMENT_DB; builder.indexing.tasklimit = 255; + builder.indexing.threads = indexing_threads; return builder; } @@ -39,6 +39,13 @@ expect_shared_threads(uint32_t exp_threads, uint32_t cpu_cores) EXPECT_EQ(exp_threads * 16, cfg.shared_task_limit()); } +void +expect_field_writer_threads(uint32_t exp_threads, uint32_t cpu_cores, uint32_t indexing_threads = 1) +{ + auto cfg = SharedThreadingServiceConfig::make(make_proton_config(0.5, indexing_threads), HwInfo::Cpu(cpu_cores)); + EXPECT_EQ(exp_threads, cfg.field_writer_threads()); +} + TEST(SharedThreadingServiceConfigTest, shared_threads_are_derived_from_cpu_cores_and_feeding_concurrency) { expect_shared_threads(4, 1); @@ -48,6 +55,21 @@ TEST(SharedThreadingServiceConfigTest, shared_threads_are_derived_from_cpu_cores expect_shared_threads(5, 10); } +TEST(SharedThreadingServiceConfigTest, field_writer_threads_are_derived_from_cpu_cores_and_feeding_concurrency) +{ + expect_field_writer_threads(3, 1); + expect_field_writer_threads(3, 4); + expect_field_writer_threads(3, 6); + expect_field_writer_threads(4, 7); + expect_field_writer_threads(4, 8); + expect_field_writer_threads(5, 9); +} + +TEST(SharedThreadingServiceConfigTest, field_writer_threads_can_be_overridden_in_proton_config) +{ + expect_field_writer_threads(4, 1, 4); +} + class SharedThreadingServiceTest : public ::testing::Test { public: Transport transport; @@ -65,7 +87,7 @@ public: transport.transport(), bucket_executor); } SequencedTaskExecutor* field_writer() { - return dynamic_cast<SequencedTaskExecutor*>(service->field_writer()); + return dynamic_cast<SequencedTaskExecutor*>(&service->field_writer()); } }; diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 7ff5eb823ca..802d175b5af 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -94,6 +94,12 @@ struct Setup { void property(const std::string &name, const std::string &val) { properties[name] = val; } + void query_feature_type(const std::string &name, const std::string &type) { + property(fmt("vespa.type.query.%s", name.c_str()), type); + } + void query_feature_default_value(const std::string &name, const std::string &expr) { + property(fmt("query(%s)", name.c_str()), expr); + } void rank_expr(const std::string &name, const std::string &expr) { property(fmt("rankingExpression(%s).rankingScript", name.c_str()), expr); } @@ -461,6 +467,32 @@ TEST_F("require that broken fragile model without dry-run passes verification", //----------------------------------------------------------------------------- +TEST_F("require that query tensor can have default value", SimpleSetup()) { + f.query_feature_type("foo", "tensor(x[3])"); + f.query_feature_default_value("foo", "tensor(x[3])(x+1)"); + f.verify_valid({"query(foo)"}); +} + +TEST_F("require that query tensor default value must have appropriate type", SimpleSetup()) { + f.query_feature_type("foo", "tensor(y[3])"); + f.query_feature_default_value("foo", "tensor(x[3])(x+1)"); + f.verify_invalid({"query(foo)"}); +} + +TEST_F("require that query tensor default value must be a valid expression", SimpleSetup()) { + f.query_feature_type("foo", "tensor(x[3])"); + f.query_feature_default_value("foo", "this expression is not parseable"); + f.verify_invalid({"query(foo)"}); +} + +TEST_F("require that query tensor default value expression does not need parameters", SimpleSetup()) { + f.query_feature_type("foo", "tensor(x[3])"); + f.query_feature_default_value("foo", "externalSymbol"); + f.verify_invalid({"query(foo)"}); +} + +//----------------------------------------------------------------------------- + TEST_F("cleanup files", Setup()) { ASSERT_TRUE(vespalib::Process::run(fmt("rm -rf %s", gen_dir.c_str()))); } diff --git a/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt b/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt index 60d3a05502a..0489690c7c4 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/bmcluster/CMakeLists.txt @@ -49,8 +49,8 @@ vespa_add_library(searchcore_bmcluster STATIC searchcore_grouping searchcore_proton_metrics searchcore_fconfig + searchcorespi storageserver_storageapp messagebus_messagebus-test messagebus - searchlib_searchlib_uca ) diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index 4db46ead525..14c96faef42 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -202,7 +202,7 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume schema, std::make_shared<proton::DocumentDBMaintenanceConfig>(), search::LogDocumentStore::Config(), - std::make_shared<const proton::ThreadingServiceConfig>(proton::ThreadingServiceConfig::make(1)), + std::make_shared<const proton::ThreadingServiceConfig>(proton::ThreadingServiceConfig::make()), std::make_shared<const proton::AllocConfig>(), "client", doc_type_name.getName()); diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 51536299d51..bb664ea1743 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -129,7 +129,8 @@ indexing.write.io enum {NORMAL, OSYNC, DIRECTIO} default=DIRECTIO restart ## Control io options during read both under dump and fusion. indexing.read.io enum {NORMAL, DIRECTIO} default=DIRECTIO restart -## Control number of threads used for indexing +## Overrides the number of threads used for writing fields across all document dbs. +## See feeding.concurrency for details. indexing.threads int default=1 restart ## Option to specify what is most important during indexing. @@ -293,17 +294,6 @@ documentdb[].configid string documentdb[].visibilitydelay double default=0.0 ## Whether this document type is globally distributed or not. documentdb[].global bool default=false -## A number between 0.0 and 1.0 that specifies the concurrency when handling feed operations. -## When set to 1.0 all cores on the cpu is utilized. -## -## 3 thread pools used for various aspect of feeding are configured based on this setting: -## 1) Writing changes to attribute fields -## 2) Inverting index fields -## 3) Writing changes to index fields -## -## The number of threads in each of pools is calculated as: -## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) -documentdb[].feeding.concurrency double default=0.2 ## Minimum initial size for any per document tables. documentdb[].allocation.initialnumdocs long default=1024 @@ -493,29 +483,17 @@ hwinfo.cpu.cores int default = 0 restart ## A number between 0.0 and 1.0 that specifies the concurrency when handling feed operations. ## When set to 1.0 all cores on the cpu is utilized. ## -## 4 thread pools used for various aspect of feeding are configured based on this setting: -## 1) Compressing and compacting documents -## 2) Writing changes to attribute fields -## 3) Inverting index fields -## 4) Writing changes to index fields +## 3 thread pools used for various aspect of feeding are configured based on this setting: +## 1) Basic shared thread pool. E.g. used for compressing and compacting documents. +## 2) Warmup thread pool. Used for disk index warmup. +## 3) Field writer thread pool. Used for writing data to document fields: +## - Inverting index fields +## - Writing changes to index fields +## - Writing changes to attribute fields ## -## The number of threads in pool 1 is calculated as: -## max(ceil(hwinfo.cpu.cores * feeding.concurrency), summary.log.numthreads) -## The number of threads in each of pools 2-4 is calculated as: -## max(ceil((hwinfo.cpu.cores * feeding.concurrency)/3), indexing.threads) -## Deprecated -> Use documentdb.feeding.concurrency +## See shared_threading_service_config.cpp for details on how the thread pool sizes are calculated. feeding.concurrency double default = 0.2 restart -## Whether we should use a shared field writer executor in the document db threading service: -## -## NONE: Don't use a shared executor. -## INDEX: Use a shared executor for index field inverter and index field writer. -## INDEX_AND_ATTRIBUTE: Use a shared executor for index field inverter, index field writer, and attribute field writer. -## DOCUMENT_DB: Use a shared executor for index field inverter, index field writer, and attribute field writer among all document dbs. -## -## TODO: Remove this when a shared executor is the default. -feeding.shared_field_writer_executor enum {NONE, INDEX, INDEX_AND_ATTRIBUTE, DOCUMENT_DB} default = DOCUMENT_DB restart - ## Maximum number of pending tasks for the master thread in each document db. ## ## This limit is only considered when executing tasks for handling external feed operations. diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h index 3d9dbda0cbc..3de4779c77a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "i_attribute_manager.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -19,10 +19,9 @@ public: AttributeManagerExplorer(const proton::IAttributeManager::SP &mgr); ~AttributeManagerExplorer(); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; - virtual std::vector<vespalib::string> get_children_names() const override; - virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + std::vector<vespalib::string> get_children_names() const override; + std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h index f0ad134d2f4..204a81ed629 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_vector_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "exclusive_attribute_read_accessor.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -19,7 +19,7 @@ public: AttributeVectorExplorer(ExclusiveAttributeReadAccessor::UP attribute); // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h index 898a8579255..71448e6fad1 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucket_db_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "bucket_db_owner.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h b/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h index 1ab1adb1add..bc6d0890906 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/remove_batch_entry.h @@ -4,7 +4,7 @@ #include <vespa/document/base/globalid.h> #include <vespa/document/bucket/bucketid.h> -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> namespace proton::bucketdb { diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt index 3f33871a0e2..25e9a469f93 100644 --- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt @@ -25,6 +25,7 @@ vespa_add_library(searchcore_pcommon STATIC selectpruner.cpp state_reporter_utils.cpp statusreport.cpp + scheduledexecutor.cpp DEPENDS searchcore_proton_metrics searchcore_fconfig diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp index 6d69c884c90..ea2d2de8b41 100644 --- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp @@ -7,6 +7,7 @@ #include <vespa/searchcore/config/config-hwinfo.h> #include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/time.h> +#include <vespa/vespalib/util/resource_limits.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/alloc.h> #include <filesystem> @@ -40,21 +41,21 @@ sampleDiskSizeBytes(const std::string &pathStr, const HwInfoSampler::Config &cfg } uint64_t -sampleMemorySizeBytes(const HwInfoSampler::Config &cfg) +sampleMemorySizeBytes(const HwInfoSampler::Config &cfg, const vespalib::ResourceLimits& resource_limits) { if (cfg.memorySizeBytes != 0) { return cfg.memorySizeBytes; } - return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); + return resource_limits.memory(); } uint32_t -sampleCpuCores(const HwInfoSampler::Config &cfg) +sampleCpuCores(const HwInfoSampler::Config &cfg, const vespalib::ResourceLimits& resource_limits) { if (cfg.cpuCores != 0) { return cfg.cpuCores; } - return std::thread::hardware_concurrency(); + return resource_limits.cpu(); } std::unique_ptr<HwinfoConfig> @@ -119,11 +120,12 @@ HwInfoSampler::HwInfoSampler(const vespalib::string &path, _diskWriteSpeed(0.0) { setDiskWriteSpeed(path, config); + auto resource_limits = vespalib::ResourceLimits::create(); setup(HwInfo::Disk(sampleDiskSizeBytes(path, config), (_diskWriteSpeed < config.slowWriteSpeedLimit), config.diskShared), - HwInfo::Memory(sampleMemorySizeBytes(config)), - HwInfo::Cpu(sampleCpuCores(config))); + HwInfo::Memory(sampleMemorySizeBytes(config, resource_limits)), + HwInfo::Cpu(sampleCpuCores(config, resource_limits))); } HwInfoSampler::~HwInfoSampler() = default; diff --git a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.cpp b/searchcore/src/vespa/searchcore/proton/common/scheduledexecutor.cpp index 99254240f3c..4577477fb77 100644 --- a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.cpp +++ b/searchcore/src/vespa/searchcore/proton/common/scheduledexecutor.cpp @@ -4,7 +4,9 @@ #include <vespa/fnet/task.h> #include <vespa/fnet/transport.h> -namespace vespalib { +using vespalib::duration; + +namespace proton { using Task = vespalib::Executor::Task; @@ -30,7 +32,7 @@ public: void PerformTask() override { _task->run(); - Schedule(to_s(_interval)); + Schedule(vespalib::to_s(_interval)); } }; @@ -52,7 +54,7 @@ ScheduledExecutor::scheduleAtFixedRate(vespalib::Executor::Task::UP task, durati std::lock_guard guard(_lock); auto tTask = std::make_unique<TimerTask>(_transport.GetScheduler(), std::move(task), interval); _taskList.push_back(std::move(tTask)); - _taskList.back()->Schedule(to_s(delay)); + _taskList.back()->Schedule(vespalib::to_s(delay)); } void diff --git a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.h b/searchcore/src/vespa/searchcore/proton/common/scheduledexecutor.h index 2c29ff52432..80c8b7edd15 100644 --- a/staging_vespalib/src/vespa/vespalib/util/scheduledexecutor.h +++ b/searchcore/src/vespa/searchcore/proton/common/scheduledexecutor.h @@ -8,7 +8,7 @@ class FNET_Transport; -namespace vespalib { +namespace proton { class TimerTask; @@ -21,9 +21,11 @@ class ScheduledExecutor { private: using TaskList = std::vector<std::unique_ptr<TimerTask>>; + using duration = vespalib::duration; + using Executor = vespalib::Executor; FNET_Transport & _transport; - std::mutex _lock; - TaskList _taskList; + std::mutex _lock; + TaskList _taskList; public: /** diff --git a/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h index d3530532c36..9675911d111 100644 --- a/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/docsummary/document_store_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "isummarymanager.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -18,8 +18,7 @@ private: public: DocumentStoreExplorer(ISummaryManager::SP mgr); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt index 5f288a116b4..ff740f7a4ae 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/CMakeLists.txt @@ -20,4 +20,5 @@ vespa_add_library(searchcore_documentmetastore STATIC searchcore_attribute searchcore_bucketdb searchcore_initializer + searchcorespi ) diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h index e4aa7aec44b..978fd0b4a14 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/document_meta_store_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "i_document_meta_store_context.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -18,8 +18,7 @@ private: public: DocumentMetaStoreExplorer(IDocumentMetaStoreContext::IReadGuard::UP metaStore); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp index 4df80dba74d..46b5f051422 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.cpp @@ -486,15 +486,11 @@ DocumentMetaStore::inspect(const GlobalId &gid, uint64_t prepare_serial_num) } DocumentMetaStore::Result -DocumentMetaStore::put(const GlobalId &gid, - const BucketId &bucketId, - const Timestamp ×tamp, - uint32_t docSize, - DocId lid, - uint64_t prepare_serial_num) +DocumentMetaStore::put(const GlobalId &gid, const BucketId &bucketId, Timestamp timestamp, + uint32_t docSize, DocId lid, uint64_t prepare_serial_num) { Result res; - RawDocumentMetaData metaData(gid, bucketId, timestamp, docSize); + RawDocumentMetaData metaData(gid, bucketId, storage::spi::Timestamp(timestamp), docSize); KeyComp comp(metaData, get_unbound_meta_data_view()); auto find_key = GidToLidMapKey::make_find_key(gid); auto& itr = _gid_to_lid_map_write_itr; @@ -545,9 +541,7 @@ DocumentMetaStore::put(const GlobalId &gid, } bool -DocumentMetaStore::updateMetaData(DocId lid, - const BucketId &bucketId, - const Timestamp ×tamp) +DocumentMetaStore::updateMetaData(DocId lid, const BucketId &bucketId, Timestamp timestamp) { if (!validLid(lid)) { return false; @@ -558,12 +552,12 @@ DocumentMetaStore::updateMetaData(DocId lid, metaData.getTimestamp(), metaData.getDocSize(), bucketId.stripUnused(), - timestamp, + storage::spi::Timestamp(timestamp), metaData.getDocSize(), _subDbType); metaData.setBucketId(bucketId); std::atomic_thread_fence(std::memory_order_release); - metaData.setTimestamp(timestamp); + metaData.setTimestamp(storage::spi::Timestamp(timestamp)); return true; } @@ -1065,7 +1059,7 @@ DocumentMetaStore::getEstimatedShrinkLidSpaceGain() const BucketId DocumentMetaStore::getBucketOf(const vespalib::GenerationHandler::Guard &, uint32_t lid) const { - if (__builtin_expect(validLidFastSafe(lid, getCommittedDocIdLimit()), true)) { + if (__builtin_expect(validLidFast(lid, getCommittedDocIdLimit()), true)) { return getRawMetaData(lid).getBucketId(); } return BucketId(); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h index 6076bfb2865..adb2778c2d2 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastore.h @@ -146,7 +146,7 @@ public: static constexpr size_t minHeaderLen = 0x1000; static constexpr size_t entrySize = sizeof(uint32_t) + GlobalId::LENGTH + sizeof(uint8_t) + - sizeof(Timestamp::Type); + sizeof(Timestamp); DocumentMetaStore(BucketDBOwnerSP bucketDB, const vespalib::string & name=getFixedName(), @@ -167,9 +167,9 @@ public: * map is then re-built the same way it was originally where add() * was used to create the <lid, gid> pairs. **/ - Result put(const GlobalId &gid, const BucketId &bucketId, - const Timestamp ×tamp, uint32_t docSize, DocId lid, uint64_t prepare_serial_num) override; - bool updateMetaData(DocId lid, const BucketId &bucketId, const Timestamp ×tamp) override; + Result put(const GlobalId &gid, const BucketId &bucketId, Timestamp timestamp, + uint32_t docSize, DocId lid, uint64_t prepare_serial_num) override; + bool updateMetaData(DocId lid, const BucketId &bucketId, Timestamp timestamp) override; bool remove(DocId lid, uint64_t prepare_serial_num) override; BucketId getBucketOf(const vespalib::GenerationHandler::Guard & guard, uint32_t lid) const override; @@ -185,7 +185,7 @@ public: void move(DocId fromLid, DocId toLid, uint64_t prepare_serial_num) override; bool validButMaybeUnusedLid(DocId lid) const { return _lidAlloc.validButMaybeUnusedLid(lid); } bool validLidFast(DocId lid) const { return _lidAlloc.validLid(lid); } - bool validLidFastSafe(DocId lid, uint32_t limit) const { return _lidAlloc.validLidSafe(lid, limit); } + bool validLidFast(DocId lid, uint32_t limit) const { return _lidAlloc.validLid(lid, limit); } bool validLid(DocId lid) const override { return validLidFast(lid); } void removeBatch(const std::vector<DocId> &lidsToRemove, const DocId docIdLimit) override; const RawDocumentMetaData & getRawMetaData(DocId lid) const override { return _metaDataStore.acquire_elem_ref(lid); } @@ -254,7 +254,7 @@ public: return AttributeVector::getGenerationHandler(); } - const search::GrowableBitVector &getActiveLids() const { return _lidAlloc.getActiveLids(); } + const search::BitVector &getActiveLids() const { return _lidAlloc.getActiveLids(); } void clearDocs(DocId lidLow, DocId lidLimit, bool in_shrink_lid_space) override; diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp index 9164797b86f..9f6b1befcdd 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp @@ -223,6 +223,7 @@ DocumentMetaStoreFlushTarget::initFlush(SerialNum currentSerial, std::shared_ptr uint64_t DocumentMetaStoreFlushTarget::getApproxBytesToWriteToDisk() const { + auto guard = _dms->getGuard(); return _dms->getEstimatedSaveByteSize(); } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp index e4ad9cf8ac2..b844d5400a9 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoresaver.cpp @@ -44,7 +44,7 @@ public: uint8_t bucketUsedBits = metaData.getBucketUsedBits(); assert(BucketId::validUsedBits(bucketUsedBits)); assert((bucketUsedBits >> BucketId::CountBits) == 0); - Timestamp::Type timestamp = metaData.getTimestamp(); + Timestamp timestamp = metaData.getTimestamp(); search::BufferWriter &datWriter(_datWriter); datWriter.write(&lid, sizeof(lid)); datWriter.write(gid.get(), GlobalId::LENGTH); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h index 49849fea2c1..58f668ecc62 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/i_store.h @@ -5,7 +5,6 @@ #include "raw_document_meta_data.h" #include <vespa/document/base/globalid.h> #include <vespa/document/bucket/bucketid.h> -#include <persistence/spi/types.h> namespace proton::documentmetastore { @@ -16,10 +15,10 @@ namespace proton::documentmetastore { **/ struct IStore { - typedef uint32_t DocId; - typedef document::GlobalId GlobalId; - typedef document::BucketId BucketId; - typedef storage::spi::Timestamp Timestamp; + using DocId = uint32_t; + using GlobalId = document::GlobalId; + using BucketId = document::BucketId; + using Timestamp = uint64_t; /** * Result after lookup in the store. @@ -49,7 +48,7 @@ struct IStore } }; - virtual ~IStore() {} + virtual ~IStore() = default; /** * Inspect the meta data associated with the given gid. @@ -71,7 +70,7 @@ struct IStore **/ virtual Result put(const GlobalId &gid, const BucketId &bucketId, - const Timestamp ×tamp, + Timestamp timestamp, uint32_t docSize, DocId lid, uint64_t prepare_serial_num) = 0; @@ -83,7 +82,7 @@ struct IStore */ virtual bool updateMetaData(DocId lid, const BucketId &bucketId, - const Timestamp ×tamp) = 0; + Timestamp timestamp) = 0; /** * Removes the <lid, meta data> pair with the given lid from this diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp index 4163b59b4ca..57f8be576db 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.cpp @@ -181,7 +181,7 @@ namespace { class WhiteListBlueprint : public SimpleLeafBlueprint { private: - const search::GrowableBitVector &_activeLids; + const search::BitVector &_activeLids; mutable std::mutex _lock; mutable std::vector<search::fef::TermFieldMatchData *> _matchDataVector; @@ -193,7 +193,7 @@ private: return createFilterSearch(strict, FilterConstraint::UPPER_BOUND); } public: - WhiteListBlueprint(const search::GrowableBitVector &activeLids) + WhiteListBlueprint(const search::BitVector &activeLids) : SimpleLeafBlueprint(FieldSpecBaseList()), _activeLids(activeLids), _matchDataVector() diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h index 05795aee7d5..6118701b0dc 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lid_allocator.h @@ -68,10 +68,11 @@ public: return lid < _usedLids.size(); } bool validLid(DocId lid) const { - return (lid < _usedLids.getSizeSafe() && _usedLids.testBit(lid)); + auto &vector = _usedLids.getBitVector(); + return (lid < vector.getSizeAcquire() && vector.testBitAcquire(lid)); } - bool validLidSafe(DocId lid, uint32_t limit) const { - return (lid < limit && _usedLids.testBitSafe(lid)); + bool validLid(DocId lid, uint32_t limit) const { + return (lid < limit && _usedLids.testBitAcquire(lid)); } DocId getLowestFreeLid() const { return _freeLids.getLowest(); @@ -80,7 +81,7 @@ public: return _usedLids.getHighest(); } - const search::GrowableBitVector &getActiveLids() const { return _activeLids.getBitVector(); } + const search::BitVector &getActiveLids() const { return _activeLids.getBitVector(); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp index 27b9cfdd197..22fa4c24cd5 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.cpp @@ -25,27 +25,27 @@ LidStateVector::resizeVector(uint32_t newSize, uint32_t newCapacity) { uint32_t lowest = getLowest(); uint32_t highest = getHighest(); - assert(!_trackLowest || lowest <= _bv.size()); - assert(!_trackHighest || _bv.size() == 0 || highest < _bv.size()); - bool nolowest(lowest == _bv.size()); - if (_bv.size() > newSize) { + assert(!_trackLowest || lowest <= _bv.writer().size()); + assert(!_trackHighest || _bv.writer().size() == 0 || highest < _bv.writer().size()); + bool nolowest(lowest == _bv.writer().size()); + if (_bv.writer().size() > newSize) { _bv.shrink(newSize); } - if (_bv.capacity() < newCapacity) { + if (_bv.writer().capacity() < newCapacity) { _bv.reserve(newCapacity); } - if (_bv.size() < newSize) { + if (_bv.writer().size() < newSize) { _bv.extend(newSize); } if (_trackLowest) { - if (nolowest || lowest > _bv.size()) { - lowest = _bv.size(); + if (nolowest || lowest > _bv.writer().size()) { + lowest = _bv.writer().size(); _lowest.store(lowest, std::memory_order_relaxed); } } if (_trackHighest) { - if (highest >= _bv.size()) { - highest = _bv.size() > 0 ? _bv.getPrevTrueBit(_bv.size() - 1) : 0; + if (highest >= _bv.writer().size()) { + highest = _bv.writer().size() > 0 ? _bv.writer().getPrevTrueBit(_bv.writer().size() - 1) : 0; _highest.store(highest, std::memory_order_relaxed); } } @@ -54,38 +54,38 @@ LidStateVector::resizeVector(uint32_t newSize, uint32_t newCapacity) void LidStateVector::updateLowest(uint32_t lowest) { - lowest = _bv.getNextTrueBit(lowest); - assert(lowest <= _bv.size()); + lowest = _bv.writer().getNextTrueBit(lowest); + assert(lowest <= _bv.writer().size()); _lowest.store(lowest, std::memory_order_relaxed); } void LidStateVector::updateHighest(uint32_t highest) { - highest = _bv.getPrevTrueBit(highest); - assert(_bv.size() == 0 || highest < _bv.size()); + highest = _bv.writer().getPrevTrueBit(highest); + assert(_bv.writer().size() == 0 || highest < _bv.writer().size()); _highest.store(highest, std::memory_order_relaxed); } void LidStateVector::setBit(unsigned int idx) { - assert(idx < _bv.size()); + assert(idx < _bv.writer().size()); if (_trackLowest && idx < getLowest()) { _lowest.store(idx, std::memory_order_relaxed); } if (_trackHighest && idx > getHighest()) { _highest.store(idx, std::memory_order_relaxed); } - assert(!_bv.testBit(idx)); - _bv.setBitAndMaintainCount(idx); + assert(!_bv.writer().testBit(idx)); + _bv.writer().setBitAndMaintainCount(idx); } template <bool do_set> uint32_t LidStateVector::assert_is_not_set_then_set_bits_helper(const std::vector<uint32_t>& idxs) { - uint32_t size = _bv.size(); + uint32_t size = _bv.writer().size(); uint32_t high = 0; uint32_t low = size; for (auto idx : idxs) { @@ -93,12 +93,12 @@ LidStateVector::assert_is_not_set_then_set_bits_helper(const std::vector<uint32_ if (idx > high) { high = idx; } - assert(!_bv.testBit(idx)); + assert(!_bv.writer().testBit(idx)); if (do_set) { if (idx < low) { low = idx; } - _bv.setBitAndMaintainCount(idx); + _bv.writer().setBitAndMaintainCount(idx); } } if (do_set) { @@ -128,9 +128,9 @@ LidStateVector::set_bits(const std::vector<uint32_t>& idxs) void LidStateVector::clearBit(unsigned int idx) { - assert(idx < _bv.size()); - assert(_bv.testBit(idx)); - _bv.clearBitAndMaintainCount(idx); + assert(idx < _bv.writer().size()); + assert(_bv.writer().testBit(idx)); + _bv.writer().clearBitAndMaintainCount(idx); maybeUpdateLowest(); maybeUpdateHighest(); } @@ -141,9 +141,9 @@ LidStateVector::assert_is_set_then_clear_bits_helper(const std::vector<uint32_t> { for (auto idx : idxs) { if (do_assert) { - assert(_bv.testBit(idx)); + assert(_bv.writer().testBit(idx)); } - _bv.clearBitAndMaintainCount(idx); + _bv.writer().clearBitAndMaintainCount(idx); } maybeUpdateLowest(); maybeUpdateHighest(); diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h index 08bd93f063d..a23c3657453 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/lidstatevector.h @@ -19,13 +19,13 @@ class LidStateVector void updateHighest(uint32_t highest); void maybeUpdateLowest() { uint32_t lowest = getLowest(); - if (_trackLowest && lowest < _bv.size() && !_bv.testBit(lowest)) { + if (_trackLowest && lowest < _bv.writer().size() && !_bv.writer().testBit(lowest)) { updateLowest(lowest); } } void maybeUpdateHighest() { uint32_t highest = getHighest(); - if (_trackHighest && highest != 0 && !_bv.testBit(highest)) { + if (_trackHighest && highest != 0 && !_bv.writer().testBit(highest)) { updateHighest(highest); } } @@ -48,10 +48,9 @@ public: void clearBit(unsigned int idx); void consider_clear_bits(const std::vector<uint32_t>& idxs); void clear_bits(const std::vector<uint32_t>& idxs); - bool testBit(unsigned int idx) const { return _bv.testBit(idx); } - bool testBitSafe(unsigned int idx) const { return _bv.testBitSafe(idx); } - unsigned int size() const { return _bv.size(); } - unsigned int getSizeSafe() const { return _bv.getSizeSafe(); } + bool testBit(unsigned int idx) const { return _bv.reader().testBit(idx); } + bool testBitAcquire(unsigned int idx) const { return _bv.reader().testBitAcquire(idx); } + unsigned int size() const { return _bv.reader().size(); } unsigned int byteSize() const { return _bv.extraByteSize() + sizeof(LidStateVector); } @@ -65,14 +64,14 @@ public: */ uint32_t count() const { // Called by document db executor thread or metrics related threads - return _bv.countTrueBits(); + return _bv.reader().countTrueBits(); } unsigned int getNextTrueBit(unsigned int idx) const { - return _bv.getNextTrueBit(idx); + return _bv.reader().getNextTrueBit(idx); } - const search::GrowableBitVector &getBitVector() const { return _bv; } + const search::BitVector &getBitVector() const { return _bv.reader(); } }; } diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h index b5e512fcd9e..1fbd9b0ac62 100644 --- a/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h +++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/raw_document_meta_data.h @@ -4,7 +4,7 @@ #include <vespa/document/base/globalid.h> #include <vespa/document/bucket/bucketid.h> -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> #include <algorithm> #include <atomic> #include <cassert> diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h index da4e2ff40ea..c07b1ce2c8f 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/createbucketoperation.h @@ -13,12 +13,11 @@ class CreateBucketOperation : public FeedOperation public: CreateBucketOperation(); CreateBucketOperation(const document::BucketId &bucketId); - virtual ~CreateBucketOperation() {} + ~CreateBucketOperation() override = default; const document::BucketId &getBucketId() const { return _bucketId; } - virtual void serialize(vespalib::nbostream &os) const override; - virtual void deserialize(vespalib::nbostream &is, - const document::DocumentTypeRepo &repo) override; - virtual vespalib::string toString() const override; + void serialize(vespalib::nbostream &os) const override; + void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; + vespalib::string toString() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h index e22e47c9932..cce84a63dfa 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/deletebucketoperation.h @@ -13,7 +13,7 @@ class DeleteBucketOperation : public RemoveDocumentsOperation public: DeleteBucketOperation(); DeleteBucketOperation(const document::BucketId &bucketId); - virtual ~DeleteBucketOperation() {} + ~DeleteBucketOperation() override = default; const document::BucketId &getBucketId() const { return _bucketId; } virtual void serialize(vespalib::nbostream &os) const override; virtual void deserialize(vespalib::nbostream &is, diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp index 6c65d7f22b8..a3b8692c6d5 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.cpp @@ -62,10 +62,10 @@ DocumentOperation::assertValidBucketId(const document::GlobalId &gid) const vespalib::string DocumentOperation::docArgsToString() const { return make_string("%s, timestamp=%" PRIu64 ", dbdId=(%s), prevDbdId=(%s), " "prevMarkedAsRemoved=%s, prevTimestamp=%" PRIu64 ", serialNum=%" PRIu64, - _bucketId.toString().c_str(), _timestamp.getValue(), + _bucketId.toString().c_str(), _timestamp, _dbdId.toString().c_str(), _prevDbdId.toString().c_str(), (_prevMarkedAsRemoved ? "true" : "false"), - _prevTimestamp.getValue(), getSerialNum()); + _prevTimestamp, getSerialNum()); } void diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h index d5086e9394b..d2f60208474 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/documentoperation.h @@ -4,35 +4,17 @@ #include "feedoperation.h" #include <vespa/searchcore/proton/common/dbdocumentid.h> #include <vespa/document/bucket/bucketid.h> -#include <persistence/spi/types.h> -#include <vespa/searchlib/query/base.h> +#include <vespa/persistence/spi/types.h> namespace proton { class DocumentOperation : public FeedOperation { -protected: - document::BucketId _bucketId; - storage::spi::Timestamp _timestamp; - DbDocumentId _dbdId; - DbDocumentId _prevDbdId; - bool _prevMarkedAsRemoved; - storage::spi::Timestamp _prevTimestamp; - mutable uint32_t _serializedDocSize; // Set by serialize()/deserialize() - uint64_t _prepare_serial_num; - - DocumentOperation(Type type) noexcept; - - DocumentOperation(Type type, document::BucketId bucketId, storage::spi::Timestamp timestamp) noexcept; - - void assertValidBucketId(const document::DocumentId &docId) const; - void assertValidBucketId(const document::GlobalId &docId) const; - vespalib::string docArgsToString() const; - public: + using Timestamp = uint64_t; ~DocumentOperation() override; const document::BucketId &getBucketId() const { return _bucketId; } - storage::spi::Timestamp getTimestamp() const { return _timestamp; } + Timestamp getTimestamp() const { return _timestamp; } search::DocumentIdT getLid() const { return _dbdId.getLid(); } search::DocumentIdT getPrevLid() const { return _prevDbdId.getLid(); } @@ -77,8 +59,8 @@ public: getLid() != getPrevLid(); } - storage::spi::Timestamp getPrevTimestamp() const { return _prevTimestamp; } - void setPrevTimestamp(storage::spi::Timestamp prevTimestamp) { _prevTimestamp = prevTimestamp; } + Timestamp getPrevTimestamp() const { return _prevTimestamp; } + void setPrevTimestamp(Timestamp prevTimestamp) { _prevTimestamp = prevTimestamp; } void serialize(vespalib::nbostream &os) const override; void deserialize(vespalib::nbostream &is, const document::DocumentTypeRepo &repo) override; @@ -89,6 +71,22 @@ public: // Provided as a hook for tests. void serializeDocumentOperationOnly(vespalib::nbostream &os) const; +protected: + document::BucketId _bucketId; + Timestamp _timestamp; + DbDocumentId _dbdId; + DbDocumentId _prevDbdId; + bool _prevMarkedAsRemoved; + Timestamp _prevTimestamp; + mutable uint32_t _serializedDocSize; // Set by serialize()/deserialize() + uint64_t _prepare_serial_num; + + DocumentOperation(Type type) noexcept; + DocumentOperation(Type type, document::BucketId bucketId, uint64_t timestamp) noexcept; + + void assertValidBucketId(const document::DocumentId &docId) const; + void assertValidBucketId(const document::GlobalId &docId) const; + vespalib::string docArgsToString() const; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp index 4f8511fe2cf..c0cd12738fe 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.cpp @@ -19,7 +19,7 @@ MoveOperation::MoveOperation() MoveOperation::MoveOperation(const BucketId &bucketId, - const Timestamp ×tamp, + Timestamp timestamp, const Document::SP &doc, DbDocumentId sourceDbdId, uint32_t targetSubDbId) diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h index 0304bf9da96..3553820b21a 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/moveoperation.h @@ -15,7 +15,7 @@ public: MoveOperation(); MoveOperation(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, + Timestamp timestamp, const DocumentSP &doc, DbDocumentId sourceDbdId, uint32_t targetSubDbId); diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp index 262f882bf3d..3324a27009b 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.cpp @@ -10,7 +10,6 @@ LOG_SETUP(".proton.feedoperation.pruneremoveddocumentsoperation"); using document::DocumentTypeRepo; using search::DocumentIdT; -using storage::spi::Timestamp; using vespalib::make_string; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h index 64e553a8e1e..ba67dca5e72 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/pruneremoveddocumentsoperation.h @@ -2,7 +2,6 @@ #pragma once #include "removedocumentsoperation.h" -#include <persistence/spi/types.h> #include <vespa/searchlib/query/base.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp index 19e85df61e9..af4df0bea36 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.cpp @@ -6,7 +6,6 @@ using document::BucketId; using document::Document; using document::DocumentTypeRepo; -using storage::spi::Timestamp; using vespalib::make_string; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h index effe57636dd..623600194e4 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/putoperation.h @@ -12,9 +12,7 @@ class PutOperation : public DocumentOperation public: PutOperation(); - PutOperation(document::BucketId bucketId, - storage::spi::Timestamp timestamp, - DocumentSP doc); + PutOperation(document::BucketId bucketId, Timestamp timestamp, DocumentSP doc); ~PutOperation() override; const DocumentSP &getDocument() const { return _doc; } void assertValid() const; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp index 06ab745ce3e..0b02d366783 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.cpp @@ -7,7 +7,6 @@ using document::BucketId; using document::DocumentId; using document::GlobalId; using document::DocumentTypeRepo; -using storage::spi::Timestamp; using vespalib::make_string; namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h index 6c94fd16dc5..5613d6c6cbd 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/removeoperation.h @@ -9,7 +9,7 @@ namespace proton { class RemoveOperation : public DocumentOperation { protected: explicit RemoveOperation(Type type) : DocumentOperation(type) {} - RemoveOperation(Type type, document::BucketId bucketId, storage::spi::Timestamp timestamp) + RemoveOperation(Type type, document::BucketId bucketId, Timestamp timestamp) : DocumentOperation(type, bucketId, timestamp) {} public: @@ -23,9 +23,7 @@ class RemoveOperationWithDocId : public RemoveOperation { public: RemoveOperationWithDocId(); - RemoveOperationWithDocId(document::BucketId bucketId, - storage::spi::Timestamp timestamp, - const document::DocumentId &docId); + RemoveOperationWithDocId(document::BucketId bucketId, Timestamp timestamp, const document::DocumentId &docId); ~RemoveOperationWithDocId() override; const document::DocumentId &getDocumentId() const { return _docId; } const document::GlobalId & getGlobalId() const override { return _docId.getGlobalId(); } @@ -43,10 +41,8 @@ class RemoveOperationWithGid : public RemoveOperation { public: RemoveOperationWithGid(); - RemoveOperationWithGid(document::BucketId bucketId, - storage::spi::Timestamp timestamp, - const document::GlobalId & gid, - vespalib::stringref docType); + RemoveOperationWithGid(document::BucketId bucketId, Timestamp timestamp, + const document::GlobalId & gid, vespalib::stringref docType); ~RemoveOperationWithGid() override; const document::GlobalId & getGlobalId() const override { return _gid; } void serialize(vespalib::nbostream &os) const override; diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp index c7c28a5bdc9..fe95df9c1c2 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.cpp @@ -12,7 +12,6 @@ using document::BucketId; using document::DocumentType; using document::DocumentTypeRepo; using document::DocumentUpdate; -using storage::spi::Timestamp; using vespalib::make_string; namespace proton { @@ -30,14 +29,14 @@ UpdateOperation::UpdateOperation(Type type) UpdateOperation::UpdateOperation(Type type, const BucketId &bucketId, - const Timestamp ×tamp, DocumentUpdate::SP upd) + Timestamp timestamp, DocumentUpdate::SP upd) : DocumentOperation(type, bucketId, timestamp), _upd(std::move(upd)) { } -UpdateOperation::UpdateOperation(const BucketId &bucketId, const Timestamp ×tamp, DocumentUpdate::SP upd) +UpdateOperation::UpdateOperation(const BucketId &bucketId, Timestamp timestamp, DocumentUpdate::SP upd) : UpdateOperation(FeedOperation::UPDATE, bucketId, timestamp, std::move(upd)) { } diff --git a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h index 64fc6fb0fcd..3f835429fcf 100644 --- a/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h +++ b/searchcore/src/vespa/searchcore/proton/feedoperation/updateoperation.h @@ -16,16 +16,14 @@ private: using DocumentUpdateSP = std::shared_ptr<document::DocumentUpdate>; DocumentUpdateSP _upd; UpdateOperation(Type type, const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - DocumentUpdateSP upd); + Timestamp timestamp, DocumentUpdateSP upd); void serializeUpdate(vespalib::nbostream &os) const; void deserializeUpdate(vespalib::nbostream && is, const document::DocumentTypeRepo &repo); public: UpdateOperation(); UpdateOperation(Type type); UpdateOperation(const document::BucketId &bucketId, - const storage::spi::Timestamp ×tamp, - DocumentUpdateSP upd); + Timestamp timestamp, DocumentUpdateSP upd); ~UpdateOperation() override; const DocumentUpdateSP &getUpdate() const { return _upd; } void serialize(vespalib::nbostream &os) const override; diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt index b506e889a97..a0e69af5b1e 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/flushengine/CMakeLists.txt @@ -16,4 +16,5 @@ vespa_add_library(searchcore_flushengine STATIC tls_stats_factory.cpp tls_stats_map.cpp DEPENDS + searchcorespi ) diff --git a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h index 27c3f705dcb..0658ba450e3 100644 --- a/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/flushengine/flush_engine_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -19,8 +19,7 @@ private: public: FlushEngineExplorer(const FlushEngine &engine); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt index 7f5ece11366..ee9f0caded1 100644 --- a/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/index/CMakeLists.txt @@ -7,4 +7,5 @@ vespa_add_library(searchcore_index STATIC indexmanager.cpp memoryindexwrapper.cpp DEPENDS + searchcorespi ) diff --git a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h index 0cce9be0f85..1fe3a7092ef 100644 --- a/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h +++ b/searchcore/src/vespa/searchcore/proton/matchengine/matchengine.h @@ -6,7 +6,7 @@ #include <vespa/searchcore/proton/common/handlermap.hpp> #include <vespa/searchcore/proton/common/statusreport.h> #include <vespa/searchlib/engine/searchapi.h> -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/simple_thread_bundle.h> #include <mutex> diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index 41e2fe2105f..1c203dd1284 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -43,4 +43,5 @@ vespa_add_library(searchcore_matching STATIC viewresolver.cpp DEPENDS searchcore_grouping + searchcorespi ) diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp index 3d8d56f0150..7bf62f678ed 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.cpp @@ -70,13 +70,6 @@ extractDiversityParams(const RankSetup &rankSetup, const Properties &rankPropert AttributeLimiter::toDiversityCutoffStrategy(DiversityCutoffStrategy::lookup(rankProperties, rankSetup.getDiversityCutoffStrategy()))); } -AttributeBlueprintParams -extractAttributeBlueprintParams(const RankSetup& rank_setup, const Properties &rankProperties) -{ - return AttributeBlueprintParams(GlobalFilterLowerLimit::lookup(rankProperties, rank_setup.get_global_filter_lower_limit()), - GlobalFilterUpperLimit::lookup(rankProperties, rank_setup.get_global_filter_upper_limit())); -} - } // namespace proton::matching::<unnamed> void @@ -181,7 +174,8 @@ MatchToolsFactory(QueryLimiter & queryLimiter, const Properties & featureOverrides, bool is_search) : _queryLimiter(queryLimiter), - _requestContext(doom, attributeContext, rankProperties, extractAttributeBlueprintParams(rankSetup, rankProperties)), + _global_filter_params(extract_global_filter_params(rankSetup, rankProperties, metaStore.getNumActiveLids(), searchContext.getDocIdLimit())), + _requestContext(doom, attributeContext, rankProperties, _global_filter_params), _query(), _match_limiter(), _queryEnv(indexEnv, attributeContext, rankProperties, searchContext.getIndexes()), @@ -208,9 +202,10 @@ MatchToolsFactory(QueryLimiter & queryLimiter, trace.addEvent(4, "Perform dictionary lookups and posting lists initialization"); _query.fetchPostings(); if (is_search) { - double lower_limit = GlobalFilterLowerLimit::lookup(rankProperties, rankSetup.get_global_filter_lower_limit()); - double upper_limit = GlobalFilterUpperLimit::lookup(rankProperties, rankSetup.get_global_filter_upper_limit()); - _query.handle_global_filter(searchContext.getDocIdLimit(), lower_limit, upper_limit, trace); + _query.handle_global_filter(searchContext.getDocIdLimit(), + _global_filter_params.global_filter_lower_limit, + _global_filter_params.global_filter_upper_limit, + trace); } _query.freeze(); trace.addEvent(5, "Prepare shared state for multi-threaded rank executors"); @@ -309,6 +304,23 @@ MatchToolsFactory::get_feature_rename_map() const return _rankSetup.get_feature_rename_map(); } +AttributeBlueprintParams +MatchToolsFactory::extract_global_filter_params(const search::fef::RankSetup& rank_setup, + const search::fef::Properties& rank_properties, + uint32_t active_docids, + uint32_t docid_limit) +{ + double lower_limit = GlobalFilterLowerLimit::lookup(rank_properties, rank_setup.get_global_filter_lower_limit()); + double upper_limit = GlobalFilterUpperLimit::lookup(rank_properties, rank_setup.get_global_filter_upper_limit()); + + // Note that we count the reserved docid 0 as active. + // This ensures that when searchable-copies=1, the ratio is 1.0. + double active_hit_ratio = std::min(active_docids + 1, docid_limit) / static_cast<double>(docid_limit); + + return {lower_limit * active_hit_ratio, + upper_limit * active_hit_ratio}; +} + AttributeOperationTask::AttributeOperationTask(const RequestContext & requestContext, vespalib::stringref attribute, vespalib::stringref operation) : _requestContext(requestContext), diff --git a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h index a7d39a0c3e8..d01ea05f3f7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/match_tools.h +++ b/searchcore/src/vespa/searchcore/proton/matching/match_tools.h @@ -93,6 +93,7 @@ class MatchToolsFactory private: using IAttributeFunctor = search::attribute::IAttributeFunctor; QueryLimiter & _queryLimiter; + search::attribute::AttributeBlueprintParams _global_filter_params; RequestContext _requestContext; Query _query; MaybeMatchPhaseLimiter::UP _match_limiter; @@ -142,6 +143,19 @@ public: const RequestContext & getRequestContext() const { return _requestContext; } const StringStringMap & get_feature_rename_map() const; + + /** + * Extracts global filter parameters from the rank-profile and query. + * + * These parameters are expected to be in the range [0.0, 1.0], which matches the range of the estimated hit ratio of the query. + * When searchable-copies > 1, we must scale the parameters to match the effective range of the estimated hit ratio. + * This is done by multiplying with the active hit ratio (active docids / docid limit). + */ + static search::attribute::AttributeBlueprintParams + extract_global_filter_params(const search::fef::RankSetup& rank_setup, + const search::fef::Properties& rank_properties, + uint32_t active_docids, + uint32_t docid_limit); }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h index 19034472ea3..b9c30de6db8 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/matching/session_manager_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton::matching { @@ -18,9 +18,9 @@ private: public: SessionManagerExplorer(const SessionManager &manager) : _manager(manager) {} - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; - virtual std::vector<vespalib::string> get_children_names() const override; - virtual std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + std::vector<vespalib::string> get_children_names() const override; + std::unique_ptr<vespalib::StateExplorer> get_child(vespalib::stringref name) const override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp index 278b0c68dab..de45770ce3c 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/document_iterator.cpp @@ -214,7 +214,7 @@ public: if (doc && _fields) { document::FieldSet::stripFields(*doc, *_fields); } - _list.push_back(createDocEntry(meta.timestamp, meta.removed, std::move(doc), _defaultSerializedSize)); + _list.push_back(createDocEntry(storage::spi::Timestamp(meta.timestamp), meta.removed, std::move(doc), _defaultSerializedSize)); } } @@ -269,7 +269,7 @@ DocumentIterator::fetchCompleteSource(const IDocumentRetriever & source, Iterate for (uint32_t lid : lidsToFetch) { const search::DocumentMetaData & meta = metaData[lidIndexMap[lid]]; assert(lid == meta.lid); - list.push_back(createDocEntry(meta.timestamp, meta.removed)); + list.push_back(createDocEntry(storage::spi::Timestamp(meta.timestamp), meta.removed)); } } else { MatchVisitor visitor(matcher, metaData, lidIndexMap, _fields.get(), list, _defaultSerializedSize); diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h index cb72737479e..529b7759530 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/i_document_retriever.h @@ -2,7 +2,7 @@ #pragma once -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/read_consistency.h> #include <vespa/searchlib/common/idocumentmetastore.h> diff --git a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp index 0621612aca7..7e9f763a758 100644 --- a/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp +++ b/searchcore/src/vespa/searchcore/proton/persistenceengine/persistenceengine.cpp @@ -489,18 +489,19 @@ PersistenceEngine::get(const Bucket& b, const document::FieldSet& fields, const for (size_t i = 0; i < retrievers->size(); ++i) { IDocumentRetriever &retriever = *(*retrievers)[i]; search::DocumentMetaData meta = retriever.getDocumentMetaData(did); - if (meta.timestamp != 0 && meta.bucketId == b.getBucketId()) { + storage::spi::Timestamp timestamp(meta.timestamp); + if (timestamp != 0 && meta.bucketId == b.getBucketId()) { if (meta.removed) { - return GetResult::make_for_tombstone(meta.timestamp); + return GetResult::make_for_tombstone(timestamp); } if (document::FieldSet::Type::NONE == fields.getType()) { - return GetResult::make_for_metadata_only(meta.timestamp); + return GetResult::make_for_metadata_only(timestamp); } document::Document::UP doc = retriever.getPartialDocument(meta.lid, did, fields); if (!doc || doc->getId().getGlobalId() != meta.gid) { return GetResult(); } - return GetResult(std::move(doc), meta.timestamp); + return GetResult(std::move(doc), timestamp); } } } diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 1daacc29fcb..9e439161503 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -54,6 +54,7 @@ vespa_add_library(searchcore_server STATIC forcecommitdonetask.cpp health_adapter.cpp heart_beat_job.cpp + hw_info_explorer.cpp idocumentdbowner.cpp ifeedview.cpp ireplayconfig.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index e010240f5f8..45cac1d02e9 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -159,7 +159,7 @@ BucketMoveJob::needMove(const ScanIterator &itr) const { return noMove; } const bool wantReady = (shouldBeReady == Trinary::True) || isActive; - LOG(spam, "checkBucket(): bucket(%s), shouldBeReady(%s), active(%s)", + LOG(spam, "needMove(): bucket(%s), shouldBeReady(%s), active(%s)", itr.getBucket().toString().c_str(), toStr(shouldBeReady), toStr(isActive)); if (wantReady) { if (!hasNotReadyDocs) { @@ -241,6 +241,11 @@ BucketMoveJob::prepareMove(std::shared_ptr<BucketMoveJob> job, BucketMover::Move void BucketMoveJob::completeMove(GuardedMoveOps ops, IDestructorCallbackSP onDone) { BucketMover & mover = ops.mover(); + if (mover.cancelled()) { + LOG(spam, "completeMove(%s, mover@%p): mover already cancelled, not processing it further", + mover.getBucket().toString().c_str(), &mover); + return; + } mover.moveDocuments(std::move(ops.success()), std::move(onDone)); ops.failed().clear(); if (checkIfMoverComplete(mover)) { @@ -280,6 +285,7 @@ void BucketMoveJob::cancelBucket(BucketId bucket) { auto inFlight = _bucketsInFlight.find(bucket); if (inFlight != _bucketsInFlight.end()) { + LOG(spam, "cancelBucket(%s): cancelling existing mover %p", bucket.toString().c_str(), inFlight->second.get()); inFlight->second->cancel(); checkIfMoverComplete(*inFlight->second); } @@ -329,7 +335,7 @@ std::shared_ptr<BucketMover> BucketMoveJob::createMover(BucketId bucket, bool wantReady) { const MaintenanceDocumentSubDB &source(wantReady ? _notReady : _ready); const MaintenanceDocumentSubDB &target(wantReady ? _ready : _notReady); - LOG(debug, "checkBucket(): mover.setupForBucket(%s, source:%u, target:%u)", + LOG(debug, "createMover(): BucketMover::create(%s, source:%u, target:%u)", bucket.toString().c_str(), source.sub_db_id(), target.sub_db_id()); return BucketMover::create(bucket, &source, target.sub_db_id(), _moveHandler); } diff --git a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp index 297a9b9254f..7123edd96ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/combiningfeedview.cpp @@ -87,7 +87,7 @@ CombiningFeedView::findPrevDbdId(const document::GlobalId &gid, if (inspectRes._found) { op.setPrevDbDocumentId(DbDocumentId(subDbId, inspectRes._lid)); op.setPrevMarkedAsRemoved(subDbId == getRemFeedViewId()); - op.setPrevTimestamp(inspectRes._timestamp); + op.setPrevTimestamp(storage::spi::Timestamp(inspectRes._timestamp)); break; } } diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp index 6030fb3cceb..401de2e34a8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp @@ -1,7 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "disk_mem_usage_sampler.h" -#include <vespa/vespalib/util/scheduledexecutor.h> +#include <vespa/searchcore/proton/common/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> #include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h> #include <filesystem> @@ -15,7 +15,7 @@ DiskMemUsageSampler::DiskMemUsageSampler(FNET_Transport & transport, const std:: _path(path_in), _sampleInterval(60s), _lastSampleTime(vespalib::steady_clock::now()), - _periodicTimer(std::make_unique<vespalib::ScheduledExecutor>(transport)), + _periodicTimer(std::make_unique<ScheduledExecutor>(transport)), _lock(), _transient_usage_providers() { diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h index fa8ac48fa1f..b6ff46bc714 100644 --- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h +++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h @@ -7,11 +7,10 @@ class FNET_Transport; -namespace vespalib { class ScheduledExecutor; } - namespace proton { class ITransientResourceUsageProvider; +class ScheduledExecutor; /* * Class to sample disk and memory usage used for filtering write operations. @@ -21,7 +20,7 @@ class DiskMemUsageSampler { std::filesystem::path _path; vespalib::duration _sampleInterval; vespalib::steady_time _lastSampleTime; - std::unique_ptr<vespalib::ScheduledExecutor> _periodicTimer; + std::unique_ptr<ScheduledExecutor> _periodicTimer; std::mutex _lock; std::vector<std::shared_ptr<const ITransientResourceUsageProvider>> _transient_usage_providers; diff --git a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp index b1ab67ab5a4..c14863137e4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/docstorevalidator.cpp @@ -129,7 +129,7 @@ DocStoreValidator::performRemoves(FeedHandler & feedHandler, const search::IDocu document::Document::UP document = store.read(lid, repo); assert(document); LOG(info, "Removing document with id %s and lid %u with gid %s in bucket %s", document->getId().toString().c_str(), lid, metaData.gid.toString().c_str(), metaData.bucketId.toString().c_str()); - auto remove = std::make_unique<RemoveOperationWithGid>(metaData.bucketId, metaData.timestamp, gid, document->getType().getName()); + auto remove = std::make_unique<RemoveOperationWithGid>(metaData.bucketId, storage::spi::Timestamp(metaData.timestamp), gid, document->getType().getName()); feedHandler.performOperation(FeedToken(), std::move(remove)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h index b514a5169e7..763a541ecaa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h index 47ea6be1711..6aec86e5378 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_collection_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "documentsubdbcollection.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -18,10 +18,9 @@ private: public: DocumentSubDBCollectionExplorer(const DocumentSubDBCollection &subDbs); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; - virtual std::vector<vespalib::string> get_children_names() const override; - virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + std::vector<vespalib::string> get_children_names() const override; + std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h index f23b6b12c6c..61d0d00d69e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "idocumentsubdb.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -18,10 +18,9 @@ private: public: DocumentSubDBExplorer(const IDocumentSubDB &subDb); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; - virtual std::vector<vespalib::string> get_children_names() const override; - virtual std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + std::vector<vespalib::string> get_children_names() const override; + std::unique_ptr<StateExplorer> get_child(vespalib::stringref name) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp index e7977c7380b..7a1826fe8ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.cpp @@ -79,6 +79,7 @@ BucketMover::BucketMover(const BucketId &bucket, const MaintenanceDocumentSubDB _started(0), _completed(0), _needReschedule(false), + _cancelled(false), _allScheduled(false), _lastGidValid(false), _lastGid() @@ -138,6 +139,7 @@ BucketMover::moveDocuments(std::vector<GuardedMoveOp> moveOps, IDestructorCallba void BucketMover::cancel() { + _cancelled = true; setAllScheduled(); _needReschedule.store(true, std::memory_order_relaxed); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h index b8f1f7c732c..4dec53bc66b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentbucketmover.h @@ -4,7 +4,7 @@ #include <vespa/document/bucket/bucketid.h> #include <vespa/document/base/globalid.h> -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> #include <atomic> namespace vespalib { class IDestructorCallback; } @@ -123,6 +123,7 @@ public: const document::BucketId &getBucket() const { return _bucket; } void cancel(); + [[nodiscard]] bool cancelled() const noexcept { return _cancelled; } void setAllScheduled() { _allScheduled = true; } /// Signals all documents have been scheduled for move bool allScheduled() const { return _allScheduled; } @@ -147,6 +148,7 @@ private: std::atomic<uint32_t> _started; std::atomic<uint32_t> _completed; std::atomic<bool> _needReschedule; + bool _cancelled; bool _allScheduled; // All moves started, or operation has been cancelled bool _lastGidValid; document::GlobalId _lastGid; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 8cc64f1b0de..bb6d45ac482 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -248,18 +248,6 @@ find_document_db_config_entry(const ProtonConfig::DocumentdbVector& document_dbs return default_document_db_config_entry; } -std::shared_ptr<const ThreadingServiceConfig> -build_threading_service_config(const ProtonConfig &proton_config, - const HwInfo &hw_info, - const vespalib::string& doc_type_name) -{ - auto& document_db_config_entry = find_document_db_config_entry(proton_config.documentdb, doc_type_name); - return std::make_shared<const ThreadingServiceConfig> - (ThreadingServiceConfig::make(proton_config, - document_db_config_entry.feeding.concurrency, - hw_info.cpu())); -} - std::shared_ptr<const AllocConfig> build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& doc_type_name) { @@ -420,7 +408,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot if (newMaintenanceConfig && oldMaintenanceConfig && (*newMaintenanceConfig == *oldMaintenanceConfig)) { newMaintenanceConfig = oldMaintenanceConfig; } - auto new_threading_service_config = build_threading_service_config(_bootstrapConfig->getProtonConfig(), _bootstrapConfig->getHwInfo(), _docTypeName); + auto new_threading_service_config = std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(_bootstrapConfig->getProtonConfig())); if (new_threading_service_config && old_threading_service_config && (*new_threading_service_config == *old_threading_service_config)) { new_threading_service_config = old_threading_service_config; diff --git a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h index f0bb20ab64e..46071027855 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/executor_threading_service_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace searchcorespi::index { struct IThreadingService; } namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index dd735c75d79..8c73067056d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -15,7 +15,6 @@ using vespalib::SingleExecutor; using vespalib::SyncableThreadExecutor; using vespalib::steady_time; using OptimizeFor = vespalib::Executor::OptimizeFor; -using SharedFieldWriterExecutor = proton::ThreadingServiceConfig::SharedFieldWriterExecutor; namespace proton { @@ -34,22 +33,20 @@ createExecutorWithOneThread(uint32_t stackSize, uint32_t taskLimit, OptimizeFor VESPA_THREAD_STACK_TAG(master_executor) VESPA_THREAD_STACK_TAG(index_executor) VESPA_THREAD_STACK_TAG(summary_executor) -VESPA_THREAD_STACK_TAG(index_field_inverter_executor) -VESPA_THREAD_STACK_TAG(index_field_writer_executor) -VESPA_THREAD_STACK_TAG(attribute_field_writer_executor) -VESPA_THREAD_STACK_TAG(field_writer_executor) } -ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor &sharedExecutor, FNET_Transport & transport, - const vespalib::Clock & clock, uint32_t num_treads) - : ExecutorThreadingService(sharedExecutor, transport, clock, nullptr, nullptr, ThreadingServiceConfig::make(num_treads)) +ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor& sharedExecutor, + FNET_Transport& transport, + const vespalib::Clock& clock, + vespalib::ISequencedTaskExecutor& field_writer) + : ExecutorThreadingService(sharedExecutor, transport, clock, field_writer, nullptr, ThreadingServiceConfig::make()) {} ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor & sharedExecutor, FNET_Transport & transport, const vespalib::Clock & clock, - vespalib::ISequencedTaskExecutor * field_writer, + vespalib::ISequencedTaskExecutor& field_writer, vespalib::InvokeService * invokerService, const ThreadingServiceConfig & cfg, uint32_t stackSize) @@ -58,7 +55,6 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor & sharedEx _transport(transport), _clock(clock), _masterExecutor(1, stackSize, CpuUsage::wrap(master_executor, CpuUsage::Category::WRITE)), - _shared_field_writer(cfg.shared_field_writer()), _master_task_limit(cfg.master_task_limit()), _indexExecutor(createExecutorWithOneThread(stackSize, cfg.defaultTaskLimit(), cfg.optimize(), CpuUsage::wrap(index_executor, CpuUsage::Category::WRITE))), @@ -66,62 +62,15 @@ ExecutorThreadingService::ExecutorThreadingService(vespalib::Executor & sharedEx CpuUsage::wrap(summary_executor, CpuUsage::Category::WRITE))), _masterService(_masterExecutor), _indexService(*_indexExecutor), - _indexFieldInverter(), - _indexFieldWriter(), - _attributeFieldWriter(), - _field_writer(), - _index_field_inverter_ptr(), - _index_field_writer_ptr(), - _attribute_field_writer_ptr(), + _index_field_inverter(field_writer), + _index_field_writer(field_writer), + _attribute_field_writer(field_writer), _invokeRegistrations() { if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_indexExecutor.get()](){ executor->wakeup();})); _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_summaryExecutor.get()](){ executor->wakeup();})); } - if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) { - _field_writer = SequencedTaskExecutor::create(CpuUsage::wrap(field_writer_executor, CpuUsage::Category::WRITE), - cfg.indexingThreads() * 2, cfg.defaultTaskLimit()); - _attributeFieldWriter = SequencedTaskExecutor::create(CpuUsage::wrap(attribute_field_writer_executor, CpuUsage::Category::WRITE), - cfg.indexingThreads(), cfg.defaultTaskLimit(), - cfg.is_task_limit_hard(), cfg.optimize(), cfg.kindOfwatermark()); - if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { - _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_attributeFieldWriter.get()](){ executor->wakeup();})); - } - _index_field_inverter_ptr = _field_writer.get(); - _index_field_writer_ptr = _field_writer.get(); - _attribute_field_writer_ptr = _attributeFieldWriter.get(); - - } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { - _field_writer = SequencedTaskExecutor::create(CpuUsage::wrap(field_writer_executor, CpuUsage::Category::WRITE), - cfg.indexingThreads() * 3, cfg.defaultTaskLimit(), - cfg.is_task_limit_hard(), cfg.optimize(), cfg.kindOfwatermark()); - if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { - _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_field_writer.get()](){ executor->wakeup();})); - } - _index_field_inverter_ptr = _field_writer.get(); - _index_field_writer_ptr = _field_writer.get(); - _attribute_field_writer_ptr = _field_writer.get(); - } else if (_shared_field_writer == SharedFieldWriterExecutor::DOCUMENT_DB) { - assert(field_writer != nullptr); - _index_field_inverter_ptr = field_writer; - _index_field_writer_ptr = field_writer; - _attribute_field_writer_ptr = field_writer; - } else { - _indexFieldInverter = SequencedTaskExecutor::create(CpuUsage::wrap(index_field_inverter_executor, CpuUsage::Category::WRITE), - cfg.indexingThreads(), cfg.defaultTaskLimit()); - _indexFieldWriter = SequencedTaskExecutor::create(CpuUsage::wrap(index_field_writer_executor, CpuUsage::Category::WRITE), - cfg.indexingThreads(), cfg.defaultTaskLimit()); - _attributeFieldWriter = SequencedTaskExecutor::create(CpuUsage::wrap(attribute_field_writer_executor, CpuUsage::Category::WRITE), - cfg.indexingThreads(), cfg.defaultTaskLimit(), - cfg.is_task_limit_hard(), cfg.optimize(), cfg.kindOfwatermark()); - if (cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT && invokerService) { - _invokeRegistrations.push_back(invokerService->registerInvoke([executor=_attributeFieldWriter.get()](){ executor->wakeup();})); - } - _index_field_inverter_ptr = _indexFieldInverter.get(); - _index_field_writer_ptr = _indexFieldWriter.get(); - _attribute_field_writer_ptr = _attributeFieldWriter.get(); - } } ExecutorThreadingService::~ExecutorThreadingService() = default; @@ -140,11 +89,11 @@ void ExecutorThreadingService::shutdown() { _masterExecutor.shutdown().sync(); - _attribute_field_writer_ptr->sync_all(); + _attribute_field_writer.sync_all(); _summaryExecutor->shutdown().sync(); _indexExecutor->shutdown().sync(); - _index_field_inverter_ptr->sync_all(); - _index_field_writer_ptr->sync_all(); + _index_field_inverter.sync_all(); + _index_field_writer.sync_all(); } void @@ -156,9 +105,9 @@ ExecutorThreadingService::set_task_limits(uint32_t master_task_limit, _indexExecutor->setTaskLimit(field_task_limit); _summaryExecutor->setTaskLimit(summary_task_limit); // TODO: Move this to a common place when the field writer is always shared. - _index_field_inverter_ptr->setTaskLimit(field_task_limit); - _index_field_writer_ptr->setTaskLimit(field_task_limit); - _attribute_field_writer_ptr->setTaskLimit(field_task_limit); + _index_field_inverter.setTaskLimit(field_task_limit); + _index_field_writer.setTaskLimit(field_task_limit); + _attribute_field_writer.setTaskLimit(field_task_limit); } ExecutorThreadingServiceStats @@ -167,44 +116,25 @@ ExecutorThreadingService::getStats() auto master_stats = _masterExecutor.getStats(); auto index_stats = _indexExecutor->getStats(); auto summary_stats = _summaryExecutor->getStats(); - if (_shared_field_writer == SharedFieldWriterExecutor::INDEX) { - auto field_writer_stats = _field_writer->getStats(); - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, - field_writer_stats, - field_writer_stats, - _attribute_field_writer_ptr->getStats()); - } else if (_shared_field_writer == SharedFieldWriterExecutor::INDEX_AND_ATTRIBUTE) { - auto field_writer_stats = _field_writer->getStats(); - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, - field_writer_stats, - field_writer_stats, - field_writer_stats); - } else if (_shared_field_writer == SharedFieldWriterExecutor::DOCUMENT_DB) { - vespalib::ExecutorStats empty_stats; - // In this case the field writer stats are reported at a higher level. - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, - empty_stats, empty_stats, empty_stats); - } else { - return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, - _index_field_inverter_ptr->getStats(), - _index_field_writer_ptr->getStats(), - _attribute_field_writer_ptr->getStats()); - } + vespalib::ExecutorStats empty_stats; + // In this case the field writer stats are reported at a higher level. + return ExecutorThreadingServiceStats(master_stats, index_stats, summary_stats, + empty_stats, empty_stats, empty_stats); } vespalib::ISequencedTaskExecutor & ExecutorThreadingService::indexFieldInverter() { - return *_index_field_inverter_ptr; + return _index_field_inverter; } vespalib::ISequencedTaskExecutor & ExecutorThreadingService::indexFieldWriter() { - return *_index_field_writer_ptr; + return _index_field_writer; } vespalib::ISequencedTaskExecutor & ExecutorThreadingService::attributeFieldWriter() { - return *_attribute_field_writer_ptr; + return _attribute_field_writer; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 1179c88ef76..7c8056b816c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -24,19 +24,14 @@ private: FNET_Transport & _transport; const vespalib::Clock & _clock; vespalib::ThreadStackExecutor _masterExecutor; - ThreadingServiceConfig::SharedFieldWriterExecutor _shared_field_writer; std::atomic<uint32_t> _master_task_limit; std::unique_ptr<vespalib::SyncableThreadExecutor> _indexExecutor; std::unique_ptr<vespalib::SyncableThreadExecutor> _summaryExecutor; SyncableExecutorThreadService _masterService; ExecutorThreadService _indexService; - std::unique_ptr<vespalib::ISequencedTaskExecutor> _indexFieldInverter; - std::unique_ptr<vespalib::ISequencedTaskExecutor> _indexFieldWriter; - std::unique_ptr<vespalib::ISequencedTaskExecutor> _attributeFieldWriter; - std::unique_ptr<vespalib::ISequencedTaskExecutor> _field_writer; - vespalib::ISequencedTaskExecutor* _index_field_inverter_ptr; - vespalib::ISequencedTaskExecutor* _index_field_writer_ptr; - vespalib::ISequencedTaskExecutor* _attribute_field_writer_ptr; + vespalib::ISequencedTaskExecutor& _index_field_inverter; + vespalib::ISequencedTaskExecutor& _index_field_writer; + vespalib::ISequencedTaskExecutor& _attribute_field_writer; std::vector<Registration> _invokeRegistrations; public: @@ -44,13 +39,15 @@ public: /** * Convenience constructor used in unit tests. */ - ExecutorThreadingService(vespalib::Executor& sharedExecutor, FNET_Transport & transport, - const vespalib::Clock & clock, uint32_t num_treads = 1); + ExecutorThreadingService(vespalib::Executor& sharedExecutor, + FNET_Transport& transport, + const vespalib::Clock& clock, + vespalib::ISequencedTaskExecutor& field_writer); ExecutorThreadingService(vespalib::Executor& sharedExecutor, FNET_Transport & transport, const vespalib::Clock & clock, - vespalib::ISequencedTaskExecutor* field_writer, + vespalib::ISequencedTaskExecutor& field_writer, vespalib::InvokeService * invokeService, const ThreadingServiceConfig& cfg, uint32_t stackSize = 128 * 1024); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp index 402de8ce7ea..9ac5b02c61e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.cpp @@ -34,7 +34,6 @@ using document::DocumentTypeRepo; using storage::spi::RemoveResult; using storage::spi::Result; using storage::spi::Timestamp; -using storage::spi::Timestamp; using storage::spi::UpdateResult; using vespalib::Executor; using vespalib::IllegalStateException; @@ -208,7 +207,7 @@ FeedHandler::performInternalUpdate(FeedToken token, UpdateOperation &op) { appendOperation(op, token); if (token) { - token->setResult(make_unique<UpdateResult>(op.getPrevTimestamp()), true); + token->setResult(make_unique<UpdateResult>(Timestamp(op.getPrevTimestamp())), true); } _activeFeedView->handleUpdate(std::move(token), op); } @@ -225,7 +224,7 @@ FeedHandler::createNonExistingDocument(FeedToken token, const UpdateOperation &o _activeFeedView->preparePut(putOp); appendOperation(putOp, token); if (token) { - token->setResult(make_unique<UpdateResult>(putOp.getTimestamp()), true); + token->setResult(make_unique<UpdateResult>(Timestamp(putOp.getTimestamp())), true); } _activeFeedView->handlePut(feedtoken::make(std::make_unique<DaisyChainedFeedToken>(std::move(token))), putOp); diff --git a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h index 4e9c016af9b..1de6eb79b63 100644 --- a/searchcore/src/vespa/searchcore/proton/server/feedhandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/feedhandler.h @@ -11,7 +11,6 @@ #include "ipruneremoveddocumentshandler.h" #include "tlswriter.h" #include "transactionlogmanager.h" -#include <persistence/spi/types.h> #include <vespa/document/bucket/bucketid.h> #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/searchcore/proton/common/feedtoken.h> @@ -58,7 +57,6 @@ private: using Packet = search::transactionlog::Packet; using RPC = search::transactionlog::client::RPC; using SerialNum = search::SerialNum; - using Timestamp = storage::spi::Timestamp; using BucketId = document::BucketId; using FeedStateSP = std::shared_ptr<FeedState>; using FeedOperationUP = std::unique_ptr<FeedOperation>; diff --git a/searchcore/src/vespa/searchcore/proton/server/health_adapter.h b/searchcore/src/vespa/searchcore/proton/server/health_adapter.h index 59819400027..98abefe3860 100644 --- a/searchcore/src/vespa/searchcore/proton/server/health_adapter.h +++ b/searchcore/src/vespa/searchcore/proton/server/health_adapter.h @@ -3,7 +3,7 @@ #pragma once #include <vespa/searchcore/proton/common/statusreport.h> -#include <vespa/vespalib/net/health_producer.h> +#include <vespa/vespalib/net/http/health_producer.h> namespace proton { @@ -14,7 +14,7 @@ private: public: HealthAdapter(const StatusProducer &sp); - virtual Health getHealth() const override; + Health getHealth() const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/hw_info_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/hw_info_explorer.cpp new file mode 100644 index 00000000000..131b31a74eb --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/hw_info_explorer.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "hw_info_explorer.h" +#include <vespa/vespalib/data/slime/cursor.h> + +namespace proton { + +HwInfoExplorer::HwInfoExplorer(const HwInfo& info) + : _info(info) +{ +} + +void +HwInfoExplorer::get_state(const vespalib::slime::Inserter& inserter, bool full) const +{ + auto& object = inserter.insertObject(); + if (full) { + auto& disk = object.setObject("disk"); + disk.setLong("size_bytes", _info.disk().sizeBytes()); + disk.setBool("slow", _info.disk().slow()); + disk.setBool("shared", _info.disk().shared()); + + auto& memory = object.setObject("memory"); + memory.setLong("size_bytes", _info.memory().sizeBytes()); + + auto& cpu = object.setObject("cpu"); + cpu.setLong("cores", _info.cpu().cores()); + } +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/hw_info_explorer.h b/searchcore/src/vespa/searchcore/proton/server/hw_info_explorer.h new file mode 100644 index 00000000000..f374bc0d678 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/hw_info_explorer.h @@ -0,0 +1,24 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/vespalib/net/http/state_explorer.h> + +namespace proton { + +/** + * Class used to explore the hardware information on the machine on which proton runs. + */ +class HwInfoExplorer : public vespalib::StateExplorer +{ +private: + HwInfo _info; + +public: + HwInfoExplorer(const HwInfo& info); + + void get_state(const vespalib::slime::Inserter& inserter, bool full) const override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h index dccea41373f..0bfd874b90f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/i_shared_threading_service.h @@ -39,11 +39,8 @@ public: /** * Returns the sequenced executor used to write index and attribute fields in a document db. - * - * This is a nullptr if the field writer is not shared across all document dbs. - * TODO: Make this a reference when it is always shared. */ - virtual vespalib::ISequencedTaskExecutor* field_writer() = 0; + virtual vespalib::ISequencedTaskExecutor& field_writer() = 0; /** * Returns an InvokeService intended for regular wakeup calls. diff --git a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp index 84053786f69..13a70526789 100644 --- a/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/lid_space_compaction_handler.cpp @@ -75,7 +75,7 @@ LidSpaceCompactionHandler::createMoveOperation(const search::DocumentMetaData &d return MoveOperation::UP(); } auto doc = _subDb.retriever()->getFullDocument(moveFromLid); - auto op = std::make_unique<MoveOperation>(document.bucketId, document.timestamp, + auto op = std::make_unique<MoveOperation>(document.bucketId, storage::spi::Timestamp(document.timestamp), std::move(doc), DbDocumentId(_subDb.sub_db_id(), moveFromLid), _subDb.sub_db_id()); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h index e7b28232df5..76aef09e58c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_controller_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "maintenancejobrunner.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp index 0df211b5a0b..5ca47d4d800 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.cpp @@ -5,8 +5,8 @@ #include "document_db_maintenance_config.h" #include "i_blockable_maintenance_job.h" #include <vespa/searchcorespi/index/i_thread_service.h> +#include <vespa/searchcore/proton/common/scheduledexecutor.h> #include <vespa/vespalib/util/lambdatask.h> -#include <vespa/vespalib/util/scheduledexecutor.h> #include <vespa/fastos/thread.h> #include <thread> @@ -51,7 +51,7 @@ MaintenanceController::MaintenanceController(FNET_Transport & transport, _readySubDB(), _remSubDB(), _notReadySubDB(), - _periodicTimer(std::make_unique<vespalib::ScheduledExecutor>(transport)), + _periodicTimer(std::make_unique<ScheduledExecutor>(transport)), _config(), _state(State::INITIALIZING), _docTypeName(docTypeName), diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h index 8e5bb8d860c..763225045b2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h +++ b/searchcore/src/vespa/searchcore/proton/server/maintenancecontroller.h @@ -6,9 +6,9 @@ #include "i_maintenance_job.h" #include <vespa/searchcore/proton/common/doctypename.h> #include <vespa/vespalib/util/retain_guard.h> -#include <vespa/vespalib/util/scheduledexecutor.h> #include <mutex> +class FNET_Transport; namespace vespalib { @@ -26,6 +26,7 @@ namespace proton { class MaintenanceJobRunner; class DocumentDBMaintenanceConfig; +class ScheduledExecutor; /** * Class that controls the bucket moving between ready and notready sub databases @@ -88,7 +89,7 @@ private: MaintenanceDocumentSubDB _readySubDB; MaintenanceDocumentSubDB _remSubDB; MaintenanceDocumentSubDB _notReadySubDB; - std::unique_ptr<vespalib::ScheduledExecutor> _periodicTimer; + std::unique_ptr<ScheduledExecutor> _periodicTimer; DocumentDBMaintenanceConfigSP _config; State _state; const DocTypeName &_docTypeName; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index c558b749a60..c6e352e6df7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -4,6 +4,7 @@ #include "document_db_explorer.h" #include "fileconfigmanager.h" #include "flushhandlerproxy.h" +#include "hw_info_explorer.h" #include "memoryflush.h" #include "persistencehandlerproxy.h" #include "prepare_restart_handler.h" @@ -36,7 +37,7 @@ #include <vespa/searchlib/transactionlog/translogserverapp.h> #include <vespa/searchlib/util/fileheadertk.h> #include <vespa/vespalib/io/fileutil.h> -#include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/net/http/state_server.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> #include <vespa/vespalib/util/cpu_usage.h> #include <vespa/vespalib/util/host_name.h> @@ -211,6 +212,7 @@ Proton::Proton(FastOS_ThreadPool & threadPool, FNET_Transport & transport, const IPersistenceEngineOwner(), ComponentConfigProducer(), _cpu_util(), + _hw_info(), _threadPool(threadPool), _transport(transport), _configUri(configUri), @@ -275,6 +277,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) assert( _initStarted && ! _initComplete ); const ProtonConfig &protonConfig = configSnapshot->getProtonConfig(); const HwInfo & hwInfo = configSnapshot->getHwInfo(); + _hw_info = hwInfo; setBucketCheckSumType(protonConfig); setFS4Compression(protonConfig); @@ -806,9 +809,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &) if (_shared_service) { metrics.shared.update(_shared_service->shared().getStats()); metrics.warmup.update(_shared_service->warmup().getStats()); - if (_shared_service->field_writer()) { - metrics.field_writer.update(_shared_service->field_writer()->getStats()); - } + metrics.field_writer.update(_shared_service->field_writer().getStats()); } } } @@ -896,6 +897,7 @@ const vespalib::string FLUSH_ENGINE = "flushengine"; const vespalib::string TLS_NAME = "tls"; const vespalib::string RESOURCE_USAGE = "resourceusage"; const vespalib::string THREAD_POOLS = "threadpools"; +const vespalib::string HW_INFO = "hwinfo"; struct StateExplorerProxy : vespalib::StateExplorer { const StateExplorer &explorer; @@ -941,7 +943,7 @@ Proton::get_state(const vespalib::slime::Inserter &, bool) const std::vector<vespalib::string> Proton::get_children_names() const { - return {DOCUMENT_DB, THREAD_POOLS, MATCH_ENGINE, FLUSH_ENGINE, TLS_NAME, RESOURCE_USAGE}; + return {DOCUMENT_DB, THREAD_POOLS, MATCH_ENGINE, FLUSH_ENGINE, TLS_NAME, HW_INFO, RESOURCE_USAGE}; } std::unique_ptr<vespalib::StateExplorer> @@ -966,7 +968,10 @@ Proton::get_child(vespalib::stringref name) const (_flushEngine) ? &_flushEngine->get_executor() : nullptr, &_executor, (_shared_service) ? &_shared_service->warmup() : nullptr, - (_shared_service) ? _shared_service->field_writer() : nullptr); + (_shared_service) ? &_shared_service->field_writer() : nullptr); + + } else if (name == HW_INFO) { + return std::make_unique<HwInfoExplorer>(_hw_info); } return Explorer_UP(nullptr); } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.h b/searchcore/src/vespa/searchcore/proton/server/proton.h index a18d409b57d..6e154159ecc 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton.h @@ -19,11 +19,11 @@ #include <vespa/searchcore/proton/persistenceengine/ipersistenceengineowner.h> #include <vespa/searchlib/common/fileheadercontext.h> #include <vespa/searchlib/engine/monitorapi.h> -#include <vespa/vespalib/net/component_config_producer.h> -#include <vespa/vespalib/net/generic_state_handler.h> -#include <vespa/vespalib/net/json_get_handler.h> -#include <vespa/vespalib/net/json_handler_repo.h> -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/component_config_producer.h> +#include <vespa/vespalib/net/http/generic_state_handler.h> +#include <vespa/vespalib/net/http/json_get_handler.h> +#include <vespa/vespalib/net/http/json_handler_repo.h> +#include <vespa/vespalib/net/http/state_explorer.h> #include <vespa/vespalib/util/varholder.h> #include <vespa/vespalib/util/cpu_usage.h> #include <mutex> @@ -85,6 +85,7 @@ private: }; vespalib::CpuUtil _cpu_util; + HwInfo _hw_info; FastOS_ThreadPool & _threadPool; FNET_Transport & _transport; const config::ConfigUri _configUri; diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h index ddb9c1bed92..0ccf1b1a348 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_configurer.h @@ -6,7 +6,7 @@ #include "i_proton_configurer.h" #include <vespa/document/bucket/bucketspace.h> #include <vespa/searchcore/proton/common/doctypename.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/net/http/simple_component_config_producer.h> #include <vespa/vespalib/util/monitored_refcount.h> #include <map> #include <mutex> diff --git a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h index 2cacdd2c336..2891309fb89 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/proton_thread_pools_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace vespalib { class ISequencedTaskExecutor; diff --git a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h index 6198f588a32..8963f095834 100644 --- a/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h +++ b/searchcore/src/vespa/searchcore/proton/server/pruneremoveddocumentsjob.h @@ -3,7 +3,6 @@ #include "blockable_maintenance_job.h" #include "document_db_maintenance_config.h" -#include <persistence/spi/types.h> #include <vespa/document/bucket/bucketspace.h> #include <vespa/vespalib/util/retain_guard.h> #include <atomic> diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h index 0c36b19b7ad..64da36807a1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h +++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace proton { @@ -22,7 +22,7 @@ public: ResourceUsageExplorer(const DiskMemUsageFilter& usage_filter, const ResourceUsageTracker& usage_tracker); - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp index 79827ce81c0..86db96a20ac 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.cpp @@ -16,8 +16,6 @@ VESPA_THREAD_STACK_TAG(proton_warmup_executor) namespace proton { -using SharedFieldWriterExecutor = ThreadingServiceConfig::ProtonConfig::Feeding::SharedFieldWriterExecutor; - SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfig& cfg, FNET_Transport& transport, storage::spi::BucketExecutor& bucket_executor) @@ -35,18 +33,16 @@ SharedThreadingService::SharedThreadingService(const SharedThreadingServiceConfi _clock(_invokeService.nowRef()) { const auto& fw_cfg = cfg.field_writer_config(); - if (fw_cfg.shared_field_writer() == SharedFieldWriterExecutor::DOCUMENT_DB) { - _field_writer = vespalib::SequencedTaskExecutor::create(CpuUsage::wrap(proton_field_writer_executor, CpuUsage::Category::WRITE), - fw_cfg.indexingThreads() * 3, - fw_cfg.defaultTaskLimit(), - fw_cfg.is_task_limit_hard(), - fw_cfg.optimize(), - fw_cfg.kindOfwatermark()); - if (fw_cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT) { - _invokeRegistrations.push_back(_invokeService.registerInvoke([executor = _field_writer.get()]() { - executor->wakeup(); - })); - } + _field_writer = vespalib::SequencedTaskExecutor::create(CpuUsage::wrap(proton_field_writer_executor, CpuUsage::Category::WRITE), + cfg.field_writer_threads(), + fw_cfg.defaultTaskLimit(), + fw_cfg.is_task_limit_hard(), + fw_cfg.optimize(), + fw_cfg.kindOfwatermark()); + if (fw_cfg.optimize() == vespalib::Executor::OptimizeFor::THROUGHPUT) { + _invokeRegistrations.push_back(_invokeService.registerInvoke([executor = _field_writer.get()]() { + executor->wakeup(); + })); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h index ead16441da0..019b9fe0596 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service.h @@ -36,7 +36,7 @@ public: vespalib::ThreadExecutor& warmup() override { return *_warmup; } vespalib::ThreadExecutor& shared() override { return *_shared; } - vespalib::ISequencedTaskExecutor* field_writer() override { return _field_writer.get(); } + vespalib::ISequencedTaskExecutor& field_writer() override { return *_field_writer; } vespalib::InvokeService & invokeService() override { return _invokeService; } FNET_Transport & transport() override { return _transport; } storage::spi::BucketExecutor& bucket_executor() override { return _bucket_executor; } diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp index 76b7982fedd..002ac508b4a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.cpp @@ -11,39 +11,57 @@ using ProtonConfig = SharedThreadingServiceConfig::ProtonConfig; SharedThreadingServiceConfig::SharedThreadingServiceConfig(uint32_t shared_threads_in, uint32_t shared_task_limit_in, uint32_t warmup_threads_in, + uint32_t field_writer_threads_in, const ThreadingServiceConfig& field_writer_config_in) : _shared_threads(shared_threads_in), _shared_task_limit(shared_task_limit_in), _warmup_threads(warmup_threads_in), + _field_writer_threads(field_writer_threads_in), _field_writer_config(field_writer_config_in) { } namespace { -size_t +uint32_t derive_shared_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) { - size_t scaled_cores = (size_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); + uint32_t scaled_cores = (uint32_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); // We need at least 1 guaranteed free worker in order to ensure progress. - return std::max(scaled_cores, cfg.documentdb.size() + cfg.flush.maxconcurrent + 1); + return std::max(scaled_cores, (uint32_t)cfg.documentdb.size() + cfg.flush.maxconcurrent + 1); } -size_t +uint32_t derive_warmup_threads(const HwInfo::Cpu& cpu_info) { return std::max(1u, std::min(4u, cpu_info.cores()/8)); } +uint32_t +derive_field_writer_threads(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info) +{ + uint32_t scaled_cores = (size_t)std::ceil(cpu_info.cores() * cfg.feeding.concurrency); + uint32_t field_writer_threads = std::max(scaled_cores, (uint32_t)cfg.indexing.threads); + // Originally we used at least 3 threads for writing fields: + // - index field inverter + // - index field writer + // - attribute field writer + // We keep the same lower bound for similar behavior when using the shared field writer. + return std::max(field_writer_threads, 3u); +} + } SharedThreadingServiceConfig SharedThreadingServiceConfig::make(const proton::SharedThreadingServiceConfig::ProtonConfig& cfg, const proton::HwInfo::Cpu& cpu_info) { - size_t shared_threads = derive_shared_threads(cfg, cpu_info); - return proton::SharedThreadingServiceConfig(shared_threads, shared_threads * 16, derive_warmup_threads(cpu_info), - ThreadingServiceConfig::make(cfg, cfg.feeding.concurrency, cpu_info)); + uint32_t shared_threads = derive_shared_threads(cfg, cpu_info); + uint32_t field_writer_threads = derive_field_writer_threads(cfg, cpu_info); + return proton::SharedThreadingServiceConfig(shared_threads, shared_threads * 16, + derive_warmup_threads(cpu_info), + field_writer_threads, + ThreadingServiceConfig::make(cfg)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h index 1214bfa77fa..5a2468ca1ab 100644 --- a/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/shared_threading_service_config.h @@ -19,12 +19,14 @@ private: uint32_t _shared_threads; uint32_t _shared_task_limit; uint32_t _warmup_threads; + uint32_t _field_writer_threads; ThreadingServiceConfig _field_writer_config; public: SharedThreadingServiceConfig(uint32_t shared_threads_in, uint32_t shared_task_limit_in, uint32_t warmup_threads_in, + uint32_t field_writer_threads_in, const ThreadingServiceConfig& field_writer_config_in); static SharedThreadingServiceConfig make(const ProtonConfig& cfg, const HwInfo::Cpu& cpu_info); @@ -32,6 +34,7 @@ public: uint32_t shared_threads() const { return _shared_threads; } uint32_t shared_task_limit() const { return _shared_task_limit; } uint32_t warmup_threads() const { return _warmup_threads; } + uint32_t field_writer_threads() const { return _field_writer_threads; } const ThreadingServiceConfig& field_writer_config() const { return _field_writer_config; } }; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 6b1356da50e..2736a1eaa6f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -142,7 +142,11 @@ StoreOnlyDocSubDB::clearViews() { size_t StoreOnlyDocSubDB::getNumDocs() const { - return (_metaStoreCtx) ? _metaStoreCtx->get().getNumUsedLids() : 0u; + if (_metaStoreCtx) { + auto guard = _metaStoreCtx->getReadGuard(); + return guard->get().getNumUsedLids(); + } + return 0u; } size_t diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index 335d5bab8d0..a2ab9e7c925 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -10,34 +10,22 @@ using ProtonConfig = ThreadingServiceConfig::ProtonConfig; using OptimizeFor = vespalib::Executor::OptimizeFor; -ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_, - uint32_t master_task_limit_, +ThreadingServiceConfig::ThreadingServiceConfig(uint32_t master_task_limit_, int32_t defaultTaskLimit_, OptimizeFor optimize_, uint32_t kindOfWatermark_, - vespalib::duration reactionTime_, - SharedFieldWriterExecutor shared_field_writer_) - : _indexingThreads(indexingThreads_), - _master_task_limit(master_task_limit_), + vespalib::duration reactionTime_) + : _master_task_limit(master_task_limit_), _defaultTaskLimit(std::abs(defaultTaskLimit_)), _is_task_limit_hard(defaultTaskLimit_ >= 0), _optimize(optimize_), _kindOfWatermark(kindOfWatermark_), - _reactionTime(reactionTime_), - _shared_field_writer(shared_field_writer_) + _reactionTime(reactionTime_) { } namespace { -uint32_t -calculateIndexingThreads(const ProtonConfig::Indexing & indexing, double concurrency, const HwInfo::Cpu &cpuInfo) -{ - double scaledCores = cpuInfo.cores() * concurrency; - uint32_t indexingThreads = std::max((int32_t)std::ceil(scaledCores / 3), indexing.threads); - return std::max(indexingThreads, 1u); -} - OptimizeFor selectOptimization(ProtonConfig::Indexing::Optimize optimize) { using CfgOptimize = ProtonConfig::Indexing::Optimize; @@ -52,21 +40,18 @@ selectOptimization(ProtonConfig::Indexing::Optimize optimize) { } ThreadingServiceConfig -ThreadingServiceConfig::make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo) +ThreadingServiceConfig::make(const ProtonConfig& cfg) { - uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing, concurrency, cpuInfo); - return ThreadingServiceConfig(indexingThreads, - cfg.feeding.masterTaskLimit, + return ThreadingServiceConfig(cfg.feeding.masterTaskLimit, cfg.indexing.tasklimit, selectOptimization(cfg.indexing.optimize), cfg.indexing.kindOfWatermark, - vespalib::from_s(cfg.indexing.reactiontime), - cfg.feeding.sharedFieldWriterExecutor); + vespalib::from_s(cfg.indexing.reactiontime)); } ThreadingServiceConfig -ThreadingServiceConfig::make(uint32_t indexingThreads, SharedFieldWriterExecutor shared_field_writer_) { - return ThreadingServiceConfig(indexingThreads, 0, 100, OptimizeFor::LATENCY, 0, 10ms, shared_field_writer_); +ThreadingServiceConfig::make() { + return ThreadingServiceConfig(0, 100, OptimizeFor::LATENCY, 0, 10ms); } void @@ -79,14 +64,12 @@ ThreadingServiceConfig::update(const ThreadingServiceConfig& cfg) bool ThreadingServiceConfig::operator==(const ThreadingServiceConfig &rhs) const { - return _indexingThreads == rhs._indexingThreads && - _master_task_limit == rhs._master_task_limit && + return _master_task_limit == rhs._master_task_limit && _defaultTaskLimit == rhs._defaultTaskLimit && _is_task_limit_hard == rhs._is_task_limit_hard && _optimize == rhs._optimize && _kindOfWatermark == rhs._kindOfWatermark && - _reactionTime == rhs._reactionTime && - _shared_field_writer == rhs._shared_field_writer; + _reactionTime == rhs._reactionTime; } } diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h index a54c0674263..d13c7fb392f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h @@ -17,35 +17,29 @@ class ThreadingServiceConfig { public: using ProtonConfig = const vespa::config::search::core::internal::InternalProtonType; using OptimizeFor = vespalib::Executor::OptimizeFor; - using SharedFieldWriterExecutor = ProtonConfig::Feeding::SharedFieldWriterExecutor; private: - uint32_t _indexingThreads; uint32_t _master_task_limit; uint32_t _defaultTaskLimit; bool _is_task_limit_hard; OptimizeFor _optimize; uint32_t _kindOfWatermark; vespalib::duration _reactionTime; // Maximum reaction time to new tasks - SharedFieldWriterExecutor _shared_field_writer; private: - ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t master_task_limit_, int32_t defaultTaskLimit_, - OptimizeFor optimize_, uint32_t kindOfWatermark_, - vespalib::duration reactionTime_, SharedFieldWriterExecutor shared_field_writer_); + ThreadingServiceConfig(uint32_t master_task_limit_, int32_t defaultTaskLimit_, + OptimizeFor optimize_, uint32_t kindOfWatermark_, vespalib::duration reactionTime_); public: - static ThreadingServiceConfig make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo); - static ThreadingServiceConfig make(uint32_t indexingThreads, SharedFieldWriterExecutor shared_field_writer_ = SharedFieldWriterExecutor::NONE); + static ThreadingServiceConfig make(const ProtonConfig& cfg); + static ThreadingServiceConfig make(); void update(const ThreadingServiceConfig& cfg); - uint32_t indexingThreads() const { return _indexingThreads; } uint32_t master_task_limit() const { return _master_task_limit; } uint32_t defaultTaskLimit() const { return _defaultTaskLimit; } bool is_task_limit_hard() const { return _is_task_limit_hard; } OptimizeFor optimize() const { return _optimize; } uint32_t kindOfwatermark() const { return _kindOfWatermark; } vespalib::duration reactionTime() const { return _reactionTime; } - SharedFieldWriterExecutor shared_field_writer() const { return _shared_field_writer; } bool operator==(const ThreadingServiceConfig &rhs) const; }; diff --git a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt index 3a2e443bfef..5df9060ea07 100644 --- a/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/test/CMakeLists.txt @@ -8,6 +8,7 @@ vespa_add_library(searchcore_test STATIC clusterstatehandler.cpp documentdb_config_builder.cpp dummy_feed_view.cpp + dummy_flush_target.cpp mock_index_manager.cpp mock_shared_threading_service.cpp userdocumentsbuilder.cpp @@ -17,4 +18,5 @@ vespa_add_library(searchcore_test STATIC DEPENDS searchcore_server searchcore_fconfig + searchcorespi ) diff --git a/searchcore/src/vespa/searchcore/proton/test/document.h b/searchcore/src/vespa/searchcore/proton/test/document.h index 857678909fb..bdabf2937ad 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document.h +++ b/searchcore/src/vespa/searchcore/proton/test/document.h @@ -2,7 +2,7 @@ #pragma once #include <vespa/document/fieldvalue/document.h> -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> #include <vespa/searchlib/query/base.h> namespace proton::test { diff --git a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h index b9dacf61af8..60682ae90e5 100644 --- a/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/document_meta_store_observer.h @@ -61,16 +61,12 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore Result inspect(const GlobalId &gid, uint64_t prepare_serial_num) override { return _store.inspect(gid, prepare_serial_num); } - Result put(const GlobalId &gid, - const BucketId &bucketId, - const Timestamp ×tamp, - uint32_t docSize, - DocId lid, - uint64_t prepare_serial_num) override + Result put(const GlobalId &gid, const BucketId &bucketId, Timestamp timestamp, + uint32_t docSize, DocId lid, uint64_t prepare_serial_num) override { return _store.put(gid, bucketId, timestamp, docSize, lid, prepare_serial_num); } - bool updateMetaData(DocId lid, const BucketId &bucketId, const Timestamp ×tamp) override { + bool updateMetaData(DocId lid, const BucketId &bucketId, Timestamp timestamp) override { return _store.updateMetaData(lid, bucketId, timestamp); } bool remove(DocId lid, uint64_t prepare_serial_num) override { @@ -87,8 +83,7 @@ struct DocumentMetaStoreObserver : public IDocumentMetaStore bool validLid(DocId lid) const override { return _store.validLid(lid); } - void removeBatch(const std::vector<DocId> &lidsToRemove, - const DocId docIdLimit) override { + void removeBatch(const std::vector<DocId> &lidsToRemove, const DocId docIdLimit) override { _store.removeBatch(lidsToRemove, docIdLimit); } const RawDocumentMetaData &getRawMetaData(DocId lid) const override { diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index fbd3dbd2402..301f2a97d14 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -48,7 +48,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, _schema(schema), _maintenance(std::make_shared<DocumentDBMaintenanceConfig>()), _store(), - _threading_service_config(std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make(1))), + _threading_service_config(std::make_shared<const ThreadingServiceConfig>(ThreadingServiceConfig::make())), _alloc_config(std::make_shared<const AllocConfig>()), _configId(configId), _docTypeName(docTypeName) diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp new file mode 100644 index 00000000000..8915e3b367c --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.cpp @@ -0,0 +1,15 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "dummy_flush_target.h" + +namespace proton::test { + +DummyFlushTarget::DummyFlushTarget(const vespalib::string &name) noexcept + : searchcorespi::IFlushTarget(name) +{} +DummyFlushTarget::DummyFlushTarget(const vespalib::string &name, const Type &type, const Component &component) noexcept + : searchcorespi::IFlushTarget(name, type, component) +{} +DummyFlushTarget::~DummyFlushTarget() = default; + +} diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h index 3689b181c52..a9206233c9d 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_flush_target.h @@ -7,14 +7,9 @@ namespace proton::test { struct DummyFlushTarget : public searchcorespi::IFlushTarget { - DummyFlushTarget(const vespalib::string &name) noexcept - : searchcorespi::IFlushTarget(name) - {} - DummyFlushTarget(const vespalib::string &name, - const Type &type, - const Component &component) noexcept - : searchcorespi::IFlushTarget(name, type, component) - {} + DummyFlushTarget(const vespalib::string &name) noexcept; + DummyFlushTarget(const vespalib::string &name, const Type &type, const Component &component) noexcept; + ~DummyFlushTarget() override; MemoryGain getApproxMemoryGain() const override { return MemoryGain(0, 0); } DiskGain getApproxDiskGain() const override { return DiskGain(0, 0); } SerialNum getFlushedSerialNum() const override { return 0; } diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h index 00ffdc92020..e92d4362f53 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_shared_threading_service.h @@ -26,7 +26,7 @@ public: ~MockSharedThreadingService() override; ThreadExecutor& warmup() override { return _warmup; } ThreadExecutor& shared() override { return _shared; } - vespalib::ISequencedTaskExecutor* field_writer() override { return _field_writer.get(); } + vespalib::ISequencedTaskExecutor& field_writer() override { return *_field_writer; } vespalib::InvokeService & invokeService() override { return _invokeService; } FNET_Transport & transport() override { return _transport.transport(); } storage::spi::BucketExecutor& bucket_executor() override { return _bucket_executor; } diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp index 0731a3429b1..80e2622fa3b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.cpp @@ -1,12 +1,13 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "transport_helper.h" -#include <vespa/fnet/transport.h> #include <vespa/fastos/thread.h> +#include <vespa/fnet/transport.h> +#include <vespa/searchcore/proton/server/executorthreadingservice.h> +#include <vespa/vespalib/util/sequencedtaskexecutor.h> #include <vespa/vespalib/util/size_literals.h> -#include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/util/testclock.h> -#include <vespa/searchcore/proton/server/executorthreadingservice.h> +#include <vespa/vespalib/util/threadstackexecutor.h> namespace proton { @@ -31,9 +32,12 @@ Transport::shutdown() { _transport->ShutDown(true); } +VESPA_THREAD_STACK_TAG(proton_transport_and_executor_field_writer) + TransportAndExecutor::TransportAndExecutor(size_t num_threads) : Transport(), - _sharedExecutor(std::make_unique<vespalib::ThreadStackExecutor>(num_threads, 64_Ki)) + _sharedExecutor(std::make_unique<vespalib::ThreadStackExecutor>(num_threads, 64_Ki)), + _field_writer(vespalib::SequencedTaskExecutor::create(proton_transport_and_executor_field_writer, num_threads)) {} TransportAndExecutor::~TransportAndExecutor() = default; @@ -45,8 +49,9 @@ TransportAndExecutor::shutdown() { TransportAndExecutorService::TransportAndExecutorService(size_t num_threads) : TransportAndExecutor(num_threads), - _writeService(std::make_unique<ExecutorThreadingService>(shared(), transport(), clock())) + _writeService(std::make_unique<ExecutorThreadingService>(shared(), transport(), clock(), field_writer())) {} + TransportAndExecutorService::~TransportAndExecutorService() = default; searchcorespi::index::IThreadingService & diff --git a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h index 8ec4f50e3f0..46ca8131041 100644 --- a/searchcore/src/vespa/searchcore/proton/test/transport_helper.h +++ b/searchcore/src/vespa/searchcore/proton/test/transport_helper.h @@ -33,9 +33,12 @@ public: TransportAndExecutor(size_t num_threads); ~TransportAndExecutor() override; vespalib::Executor & shared() { return *_sharedExecutor; } + vespalib::ISequencedTaskExecutor& field_writer() { return *_field_writer; } void shutdown() override; private: std::unique_ptr<vespalib::Executor> _sharedExecutor; + std::unique_ptr<vespalib::ISequencedTaskExecutor> _field_writer; + }; class TransportAndExecutorService : public TransportAndExecutor { diff --git a/searchcorespi/src/vespa/searchcorespi/.gitignore b/searchcore/src/vespa/searchcorespi/.gitignore index 9d6ecd51398..9d6ecd51398 100644 --- a/searchcorespi/src/vespa/searchcorespi/.gitignore +++ b/searchcore/src/vespa/searchcorespi/.gitignore diff --git a/searchcorespi/src/vespa/searchcorespi/CMakeLists.txt b/searchcore/src/vespa/searchcorespi/CMakeLists.txt index 56608a219be..fab1d007a4f 100644 --- a/searchcorespi/src/vespa/searchcorespi/CMakeLists.txt +++ b/searchcore/src/vespa/searchcorespi/CMakeLists.txt @@ -1,8 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcorespi +vespa_add_library(searchcorespi STATIC SOURCES $<TARGET_OBJECTS:searchcorespi_flush> $<TARGET_OBJECTS:searchcorespi_index> - INSTALL lib64 DEPENDS ) diff --git a/persistencetypes/src/persistence/spi/.gitignore b/searchcore/src/vespa/searchcorespi/flush/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/persistencetypes/src/persistence/spi/.gitignore +++ b/searchcore/src/vespa/searchcorespi/flush/.gitignore diff --git a/searchcorespi/src/vespa/searchcorespi/flush/CMakeLists.txt b/searchcore/src/vespa/searchcorespi/flush/CMakeLists.txt index c7fa81ffd01..b2777d4327a 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/CMakeLists.txt +++ b/searchcore/src/vespa/searchcorespi/flush/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcorespi_flush OBJECT +vespa_add_library(searchcorespi_flush STATIC OBJECT SOURCES flushstats.cpp DEPENDS diff --git a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.cpp b/searchcore/src/vespa/searchcorespi/flush/flushstats.cpp index 28632219a28..28632219a28 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.cpp +++ b/searchcore/src/vespa/searchcorespi/flush/flushstats.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.h b/searchcore/src/vespa/searchcorespi/flush/flushstats.h index f92187b2112..f92187b2112 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/flushstats.h +++ b/searchcore/src/vespa/searchcorespi/flush/flushstats.h diff --git a/searchcorespi/src/vespa/searchcorespi/flush/flushtask.h b/searchcore/src/vespa/searchcorespi/flush/flushtask.h index a4de3f65fff..a4de3f65fff 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/flushtask.h +++ b/searchcore/src/vespa/searchcorespi/flush/flushtask.h diff --git a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h b/searchcore/src/vespa/searchcorespi/flush/iflushtarget.h index dff6041d7d5..dff6041d7d5 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/iflushtarget.h +++ b/searchcore/src/vespa/searchcorespi/flush/iflushtarget.h diff --git a/searchcorespi/src/vespa/searchcorespi/flush/lambdaflushtask.h b/searchcore/src/vespa/searchcorespi/flush/lambdaflushtask.h index 75737ce73d5..75737ce73d5 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/lambdaflushtask.h +++ b/searchcore/src/vespa/searchcorespi/flush/lambdaflushtask.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/.gitignore b/searchcore/src/vespa/searchcorespi/index/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/.gitignore +++ b/searchcore/src/vespa/searchcorespi/index/.gitignore diff --git a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt b/searchcore/src/vespa/searchcorespi/index/CMakeLists.txt index 3995eb836fd..ca33131d7f4 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/CMakeLists.txt +++ b/searchcore/src/vespa/searchcorespi/index/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(searchcorespi_index OBJECT +vespa_add_library(searchcorespi_index STATIC OBJECT SOURCES diskindexcleaner.cpp disk_indexes.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp b/searchcore/src/vespa/searchcorespi/index/disk_index_stats.cpp index 1b77061de8c..1b77061de8c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.cpp +++ b/searchcore/src/vespa/searchcorespi/index/disk_index_stats.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h b/searchcore/src/vespa/searchcorespi/index/disk_index_stats.h index 831d95e95c1..831d95e95c1 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/disk_index_stats.h +++ b/searchcore/src/vespa/searchcorespi/index/disk_index_stats.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp b/searchcore/src/vespa/searchcorespi/index/disk_indexes.cpp index 28f6a886d06..28f6a886d06 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.cpp +++ b/searchcore/src/vespa/searchcorespi/index/disk_indexes.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h b/searchcore/src/vespa/searchcorespi/index/disk_indexes.h index 842c1814faf..842c1814faf 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/disk_indexes.h +++ b/searchcore/src/vespa/searchcorespi/index/disk_indexes.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp b/searchcore/src/vespa/searchcorespi/index/diskindexcleaner.cpp index 3bed7ea8ea7..3bed7ea8ea7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.cpp +++ b/searchcore/src/vespa/searchcorespi/index/diskindexcleaner.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h b/searchcore/src/vespa/searchcorespi/index/diskindexcleaner.h index cbd3a5aa94f..cbd3a5aa94f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/diskindexcleaner.h +++ b/searchcore/src/vespa/searchcorespi/index/diskindexcleaner.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp b/searchcore/src/vespa/searchcorespi/index/eventlogger.cpp index 7a5b1bf907a..7a5b1bf907a 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.cpp +++ b/searchcore/src/vespa/searchcorespi/index/eventlogger.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.h b/searchcore/src/vespa/searchcorespi/index/eventlogger.h index 6191543dcb3..6191543dcb3 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/eventlogger.h +++ b/searchcore/src/vespa/searchcorespi/index/eventlogger.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h b/searchcore/src/vespa/searchcorespi/index/fakeindexsearchable.h index 25a8b6847e9..25a8b6847e9 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fakeindexsearchable.h +++ b/searchcore/src/vespa/searchcorespi/index/fakeindexsearchable.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.cpp b/searchcore/src/vespa/searchcorespi/index/fusionrunner.cpp index 1675b6091cf..1675b6091cf 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.cpp +++ b/searchcore/src/vespa/searchcorespi/index/fusionrunner.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.h b/searchcore/src/vespa/searchcorespi/index/fusionrunner.h index 92ad42b76ad..92ad42b76ad 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fusionrunner.h +++ b/searchcore/src/vespa/searchcorespi/index/fusionrunner.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/fusionspec.h b/searchcore/src/vespa/searchcorespi/index/fusionspec.h index 0b147140e55..0b147140e55 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/fusionspec.h +++ b/searchcore/src/vespa/searchcorespi/index/fusionspec.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h b/searchcore/src/vespa/searchcorespi/index/i_thread_service.h index f973908b62d..f973908b62d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/i_thread_service.h +++ b/searchcore/src/vespa/searchcorespi/index/i_thread_service.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/idiskindex.h b/searchcore/src/vespa/searchcorespi/index/idiskindex.h index 010e7e7727d..010e7e7727d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/idiskindex.h +++ b/searchcore/src/vespa/searchcorespi/index/idiskindex.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexcollection.cpp b/searchcore/src/vespa/searchcorespi/index/iindexcollection.cpp index 988c0084d4f..988c0084d4f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexcollection.cpp +++ b/searchcore/src/vespa/searchcorespi/index/iindexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexcollection.h b/searchcore/src/vespa/searchcorespi/index/iindexcollection.h index 1cbf994b44f..1cbf994b44f 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexcollection.h +++ b/searchcore/src/vespa/searchcorespi/index/iindexcollection.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmaintaineroperations.h b/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h index 9025b56dc27..9025b56dc27 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmaintaineroperations.h +++ b/searchcore/src/vespa/searchcorespi/index/iindexmaintaineroperations.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.cpp b/searchcore/src/vespa/searchcorespi/index/iindexmanager.cpp index 70770f6f012..70770f6f012 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.cpp +++ b/searchcore/src/vespa/searchcorespi/index/iindexmanager.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h b/searchcore/src/vespa/searchcorespi/index/iindexmanager.h index a4173b41aa5..a4173b41aa5 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/iindexmanager.h +++ b/searchcore/src/vespa/searchcorespi/index/iindexmanager.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h b/searchcore/src/vespa/searchcorespi/index/imemoryindex.h index 67d6e034080..67d6e034080 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/imemoryindex.h +++ b/searchcore/src/vespa/searchcorespi/index/imemoryindex.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h b/searchcore/src/vespa/searchcorespi/index/index_disk_dir.h index 335838ddf2e..335838ddf2e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir.h +++ b/searchcore/src/vespa/searchcorespi/index/index_disk_dir.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp b/searchcore/src/vespa/searchcorespi/index/index_disk_dir_state.cpp index ffe33d704c8..ffe33d704c8 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.cpp +++ b/searchcore/src/vespa/searchcorespi/index/index_disk_dir_state.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h b/searchcore/src/vespa/searchcorespi/index/index_disk_dir_state.h index d8b790b3960..d8b790b3960 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_disk_dir_state.h +++ b/searchcore/src/vespa/searchcorespi/index/index_disk_dir_state.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp b/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.cpp index 855f3c69bc9..855f3c69bc9 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.cpp +++ b/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.h b/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.h index 3e52199eeda..90b0bd55615 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_explorer.h +++ b/searchcore/src/vespa/searchcorespi/index/index_manager_explorer.h @@ -3,7 +3,7 @@ #pragma once #include "iindexmanager.h" -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace searchcorespi { @@ -18,8 +18,7 @@ private: public: IndexManagerExplorer(IIndexManager::SP mgr); - // Implements vespalib::StateExplorer - virtual void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; + void get_state(const vespalib::slime::Inserter &inserter, bool full) const override; }; } // namespace searchcorespi diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp b/searchcore/src/vespa/searchcorespi/index/index_manager_stats.cpp index a93934c1500..a93934c1500 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.cpp +++ b/searchcore/src/vespa/searchcorespi/index/index_manager_stats.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h b/searchcore/src/vespa/searchcorespi/index/index_manager_stats.h index 1e218a62660..1e218a62660 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_manager_stats.h +++ b/searchcore/src/vespa/searchcorespi/index/index_manager_stats.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp b/searchcore/src/vespa/searchcorespi/index/index_searchable_stats.cpp index 92de7d2d292..92de7d2d292 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.cpp +++ b/searchcore/src/vespa/searchcorespi/index/index_searchable_stats.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h b/searchcore/src/vespa/searchcorespi/index/index_searchable_stats.h index a61245ddb5d..a61245ddb5d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/index_searchable_stats.h +++ b/searchcore/src/vespa/searchcorespi/index/index_searchable_stats.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp b/searchcore/src/vespa/searchcorespi/index/indexcollection.cpp index d69f7d1b0a4..d69f7d1b0a4 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h b/searchcore/src/vespa/searchcorespi/index/indexcollection.h index d9fb8e973e1..d9fb8e973e1 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexcollection.h +++ b/searchcore/src/vespa/searchcorespi/index/indexcollection.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp b/searchcore/src/vespa/searchcorespi/index/indexdisklayout.cpp index c701d1dfb1d..c701d1dfb1d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexdisklayout.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.h b/searchcore/src/vespa/searchcorespi/index/indexdisklayout.h index 94b35936cc7..94b35936cc7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexdisklayout.h +++ b/searchcore/src/vespa/searchcorespi/index/indexdisklayout.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp index e72525d0aaa..e72525d0aaa 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.h b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.h index 2b9ecc9574b..2b9ecc9574b 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexflushtarget.h +++ b/searchcore/src/vespa/searchcorespi/index/indexflushtarget.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexfusiontarget.cpp b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp index 1df6d321f99..1df6d321f99 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexfusiontarget.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexfusiontarget.h b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.h index 7a9f44e6612..7a9f44e6612 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexfusiontarget.h +++ b/searchcore/src/vespa/searchcorespi/index/indexfusiontarget.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp index af273bc5e45..af273bc5e45 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.h index b3fb14e1c2e..b3fb14e1c2e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainer.h +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainer.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp b/searchcore/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp index 695de7b84ff..695de7b84ff 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainerconfig.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h b/searchcore/src/vespa/searchcorespi/index/indexmaintainerconfig.h index 3f890e6fa76..3f890e6fa76 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainerconfig.h +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainerconfig.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp b/searchcore/src/vespa/searchcorespi/index/indexmaintainercontext.cpp index efd7827fc3d..efd7827fc3d 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainercontext.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h b/searchcore/src/vespa/searchcorespi/index/indexmaintainercontext.h index 2c7aa4af48e..2c7aa4af48e 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmaintainercontext.h +++ b/searchcore/src/vespa/searchcorespi/index/indexmaintainercontext.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.cpp b/searchcore/src/vespa/searchcorespi/index/indexmanagerconfig.cpp index 8ba9efe2734..8ba9efe2734 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexmanagerconfig.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.h b/searchcore/src/vespa/searchcorespi/index/indexmanagerconfig.h index decb03d97e0..decb03d97e0 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexmanagerconfig.h +++ b/searchcore/src/vespa/searchcorespi/index/indexmanagerconfig.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.cpp b/searchcore/src/vespa/searchcorespi/index/indexreadutilities.cpp index 14556ddef29..14556ddef29 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexreadutilities.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.h b/searchcore/src/vespa/searchcorespi/index/indexreadutilities.h index aeafd746772..aeafd746772 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexreadutilities.h +++ b/searchcore/src/vespa/searchcorespi/index/indexreadutilities.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h b/searchcore/src/vespa/searchcorespi/index/indexsearchable.h index 609d7854351..609d7854351 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexsearchable.h +++ b/searchcore/src/vespa/searchcorespi/index/indexsearchable.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexsearchablevisitor.h b/searchcore/src/vespa/searchcorespi/index/indexsearchablevisitor.h index f85a2cf4af6..f85a2cf4af6 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexsearchablevisitor.h +++ b/searchcore/src/vespa/searchcorespi/index/indexsearchablevisitor.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp index cc2575f74d2..cc2575f74d2 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.cpp +++ b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.h b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.h index 313ab3cc1c7..313ab3cc1c7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/indexwriteutilities.h +++ b/searchcore/src/vespa/searchcorespi/index/indexwriteutilities.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/isearchableindexcollection.cpp b/searchcore/src/vespa/searchcorespi/index/isearchableindexcollection.cpp index 85e87965cb7..85e87965cb7 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/isearchableindexcollection.cpp +++ b/searchcore/src/vespa/searchcorespi/index/isearchableindexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/isearchableindexcollection.h b/searchcore/src/vespa/searchcorespi/index/isearchableindexcollection.h index efd7062af71..efd7062af71 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/isearchableindexcollection.h +++ b/searchcore/src/vespa/searchcorespi/index/isearchableindexcollection.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h b/searchcore/src/vespa/searchcorespi/index/ithreadingservice.h index c325d5ded11..c325d5ded11 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/ithreadingservice.h +++ b/searchcore/src/vespa/searchcorespi/index/ithreadingservice.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h b/searchcore/src/vespa/searchcorespi/index/memory_index_stats.h index ccc85ab4dc6..ccc85ab4dc6 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/memory_index_stats.h +++ b/searchcore/src/vespa/searchcorespi/index/memory_index_stats.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupconfig.h b/searchcore/src/vespa/searchcorespi/index/warmupconfig.h index 8582b7256bc..8582b7256bc 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupconfig.h +++ b/searchcore/src/vespa/searchcorespi/index/warmupconfig.h diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp b/searchcore/src/vespa/searchcorespi/index/warmupindexcollection.cpp index bf437dd7ee3..bf437dd7ee3 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.cpp +++ b/searchcore/src/vespa/searchcorespi/index/warmupindexcollection.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h b/searchcore/src/vespa/searchcorespi/index/warmupindexcollection.h index c2d70b4fd5c..c2d70b4fd5c 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/warmupindexcollection.h +++ b/searchcore/src/vespa/searchcorespi/index/warmupindexcollection.h diff --git a/searchcorespi/.gitignore b/searchcorespi/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/searchcorespi/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/searchcorespi/CMakeLists.txt b/searchcorespi/CMakeLists.txt deleted file mode 100644 index 0bbaa813752..00000000000 --- a/searchcorespi/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - fnet - vespalog - vespalib - searchlib - searchcommon - metrics - config_cloudconfig - fastlib_fast - configdefinitions - document - persistencetypes - - LIBS - src/vespa/searchcorespi - src/vespa/searchcorespi/flush - src/vespa/searchcorespi/index - - TESTS - src/tests/index/disk_indexes - src/tests/index/index_disk_layout -) diff --git a/searchcorespi/OWNERS b/searchcorespi/OWNERS deleted file mode 100644 index 23c5db50ab7..00000000000 --- a/searchcorespi/OWNERS +++ /dev/null @@ -1,3 +0,0 @@ -toregge -geirst -baldersheim diff --git a/searchcorespi/src/.gitignore b/searchcorespi/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/searchcorespi/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/searchcorespi/testrun/.gitignore b/searchcorespi/testrun/.gitignore deleted file mode 100644 index 1b1bf19bb72..00000000000 --- a/searchcorespi/testrun/.gitignore +++ /dev/null @@ -1,15 +0,0 @@ -/test-report.html -/test-report.html.bottom -/test-report.html.entry -/test-report.html.summary -/test-report.html.top -/test.1.plugin.desc.file.txt -/test.1.plugin.file.plugin_test.cpp.txt -/test.1.plugin.files.html -/test.1.plugin.log.file.txt -/tmp.end-time -/tmp.plugin-time -/tmp.plugin.log-control -/tmp.start-time -/test.*.*.result -/Makefile diff --git a/searchlib/CMakeLists.txt b/searchlib/CMakeLists.txt index 300900dbb77..43a8da19191 100644 --- a/searchlib/CMakeLists.txt +++ b/searchlib/CMakeLists.txt @@ -4,16 +4,12 @@ vespa_define_module( fastos vespalog vespalib - staging_vespalib vespaeval fnet configdefinitions metrics - fastlib_fast document config_cloudconfig - persistencetypes - searchcommon EXTERNAL_DEPENDS ${VESPA_GLIBC_RT_LIB} @@ -54,6 +50,8 @@ vespa_define_module( src/vespa/searchlib/transactionlog src/vespa/searchlib/uca src/vespa/searchlib/util + src/vespa/searchcommon/attribute + src/vespa/searchcommon/common APPS src/apps/docstore @@ -214,6 +212,8 @@ vespa_define_module( src/tests/rankingexpression/intrinsic_blueprint_adapter src/tests/ranksetup src/tests/ranksetup/verify_feature + src/tests/searchcommon/attribute/config + src/tests/searchcommon/schema src/tests/sort src/tests/sortresults src/tests/sortspec @@ -234,7 +234,7 @@ vespa_define_module( src/tests/vespa-fileheader-inspect ) -install_java_artifact(searchlib) +install_jar(searchlib.jar) vespa_install_script(src/main/sh/vespa-gbdt-converter bin) vespa_install_script(src/main/sh/vespa-treenet-converter bin) diff --git a/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt b/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt index ef98f43c968..3135deb57d0 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt +++ b/searchlib/src/tests/attribute/postinglistattribute/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_executable(searchlib_postinglistattribute_test_app TEST postinglistattribute_test.cpp DEPENDS searchlib + GTest::GTest ) -vespa_add_test(NAME searchlib_postinglistattribute_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/postinglistattribute_test.sh - DEPENDS searchlib_postinglistattribute_test_app) +vespa_add_test(NAME searchlib_postinglistattribute_test_app COMMAND searchlib_postinglistattribute_test_app) diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp index ae737b2d052..e58954e1e93 100644 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp +++ b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/document/update/arithmeticvalueupdate.h> @@ -11,9 +11,11 @@ #include <vespa/searchlib/attribute/multinumericpostattribute.h> #include <vespa/searchlib/attribute/singlestringpostattribute.h> #include <vespa/searchlib/attribute/multistringpostattribute.h> +#include <vespa/searchlib/common/growablebitvector.h> #include <vespa/searchlib/queryeval/executeinfo.h> #include <vespa/searchlib/parsequery/parse.h> #include <vespa/searchlib/attribute/enumstore.hpp> +#include <vespa/vespalib/io/fileutil.h> #include <vespa/vespalib/util/compress.h> #include <vespa/searchlib/fef/termfieldmatchdata.h> #include <vespa/fastos/file.h> @@ -22,6 +24,12 @@ #include <vespa/log/log.h> LOG_SETUP("postinglistattribute_test"); +namespace { + +vespalib::string tmp_dir("tmp"); + +} + using std::shared_ptr; bool @@ -72,16 +80,16 @@ assertIterator(const std::string &exp, SearchIterator &it, { std::stringstream ss; toStr(ss, it, md); - if (!EXPECT_EQUAL(exp, ss.str())) - return false; - return true; + bool retval = true; + EXPECT_EQ(exp, ss.str()) << (retval = false, ""); + return retval; } using AttributePtr = AttributeVector::SP; -class PostingListAttributeTest : public vespalib::TestApp +class PostingListAttributeTest : public ::testing::Test { -private: +protected: typedef IntegerAttribute::largeint_t largeint_t; typedef std::set<AttributeVector::DocId> DocSet; @@ -141,13 +149,13 @@ private: void checkPostingList(const VectorType & vec, const std::vector<BufferType> & values, const Range & range); template <typename BufferType> - void checkSearch(bool useBitVector, const AttributeVector & vec, const BufferType & term, uint32_t numHits, uint32_t docBegin, uint32_t docEnd); + void checkSearch(bool useBitVector, bool need_unpack, bool has_btree, bool has_bitvector, const AttributeVector & vec, const BufferType & term, uint32_t numHits, uint32_t docBegin, uint32_t docEnd); template <typename VectorType, typename BufferType> void testPostingList(const AttributePtr& ptr1, uint32_t numDocs, const std::vector<BufferType>& values); void testPostingList(); - void testPostingList(bool enableBitVector); - void testPostingList(bool enableBitVector, uint32_t numDocs, uint32_t numUniqueValues); + void testPostingList(bool enableBitVector, bool enable_only_bitvector); + void testPostingList(bool enableBitVector, bool enable_only_bitvector, uint32_t numDocs, uint32_t numUniqueValues); template <typename AttributeType, typename ValueType> void checkPostingList(AttributeType & vec, ValueType value, DocSet expected); @@ -171,8 +179,6 @@ private: void testStringFold(); void testDupValuesInIntArray(); void testDupValuesInStringArray(); -public: - int Main() override; }; template <> @@ -379,9 +385,9 @@ PostingListAttributeTest::assertSearch(const std::string &exp, StringAttribute & SearchContextPtr sc = getSearch<StringAttribute>(sa); sc->fetchPostings(queryeval::ExecuteInfo::TRUE); SearchBasePtr sb = sc->createIterator(&md, true); - if (!EXPECT_TRUE(assertIterator(exp, *sb))) - return false; - return true; + bool retval = true; + EXPECT_TRUE(assertIterator(exp, *sb)) << (retval = false, ""); + return retval; } @@ -392,9 +398,9 @@ PostingListAttributeTest::assertSearch(const std::string &exp, StringAttribute & SearchContextPtr sc = getSearch<StringAttribute, std::string>(sa, key, false); sc->fetchPostings(queryeval::ExecuteInfo::TRUE); SearchBasePtr sb = sc->createIterator(&md, true); - if (!EXPECT_TRUE(assertIterator(exp, *sb, &md))) - return false; - return true; + bool retval = true; + EXPECT_TRUE(assertIterator(exp, *sb, &md)) << (retval = false, ""); + return retval; } bool @@ -404,9 +410,9 @@ PostingListAttributeTest::assertSearch(const std::string &exp, IntegerAttribute SearchContextPtr sc = getSearch<IntegerAttribute, int32_t>(ia, key, false); sc->fetchPostings(queryeval::ExecuteInfo::TRUE); SearchBasePtr sb = sc->createIterator(&md, true); - if (!EXPECT_TRUE(assertIterator(exp, *sb, &md))) - return false; - return true; + bool retval = true; + EXPECT_TRUE(assertIterator(exp, *sb, &md)) << (retval = false, ""); + return retval; } @@ -456,32 +462,50 @@ PostingListAttributeTest::checkPostingList(const VectorType & vec, const std::ve auto find_result = dict.find_posting_list(enumStore.make_comparator(values[i]), dict.get_frozen_root()); ASSERT_TRUE(find_result.first.valid()); + bool has_bitvector = VectorType::PostingList::isBitVector(postingList.getTypeId(find_result.second)); typename VectorType::PostingList::Iterator postings; postings = postingList.begin(find_result.second); - - uint32_t doc = docBegin; uint32_t numHits(0); - for (; postings.valid(); ++postings) { - EXPECT_EQUAL(doc++, postings.getKey()); - numHits++; + bool has_btree = postings.valid(); + if (postings.valid()) { + uint32_t doc = docBegin; + for (; postings.valid(); ++postings) { + EXPECT_EQ(doc++, postings.getKey()); + numHits++; + } + EXPECT_EQ(doc, docEnd); + } else { + EXPECT_TRUE(has_bitvector && vec.getConfig().getEnableOnlyBitVector()); + numHits = postingList.getBitVectorEntry(find_result.second)->_bv->reader().countTrueBits(); } - EXPECT_EQUAL(doc, docEnd); - checkSearch(false, vec, values[i], numHits, docBegin, docEnd); - checkSearch(true, vec, values[i], numHits, docBegin, docEnd); + if (has_bitvector) { + uint32_t doc = docBegin; + uint32_t bv_num_hits = 0; + auto& bv = postingList.getBitVectorEntry(find_result.second)->_bv->reader(); + for (auto lid = bv.getFirstTrueBit(); lid < bv.size(); lid = bv.getNextTrueBit(lid + 1)) { + EXPECT_EQ(doc++, lid); + ++bv_num_hits; + } + EXPECT_EQ(doc, docEnd); + EXPECT_EQ(numHits, bv_num_hits); + } + checkSearch(false, true, has_btree, has_bitvector, vec, values[i], numHits, docBegin, docEnd); + checkSearch(true, true, has_btree, has_bitvector, vec, values[i], numHits, docBegin, docEnd); + checkSearch(false, false, has_btree, has_bitvector, vec, values[i], numHits, docBegin, docEnd); } } template <typename BufferType> void -PostingListAttributeTest::checkSearch(bool useBitVector, const AttributeVector & vec, const BufferType & term, uint32_t numHits, uint32_t docBegin, uint32_t docEnd) +PostingListAttributeTest::checkSearch(bool useBitVector, bool need_unpack, bool has_btree, bool has_bitvector, const AttributeVector & vec, const BufferType & term, uint32_t numHits, uint32_t docBegin, uint32_t docEnd) { SearchContextPtr sc = getSearch(vec, term, false, attribute::SearchContextParams().useBitVector(useBitVector)); EXPECT_FALSE( ! sc ); sc->fetchPostings(queryeval::ExecuteInfo::TRUE); size_t approx = sc->approximateHits(); - EXPECT_EQUAL(numHits, approx); + EXPECT_EQ(numHits, approx); if (docBegin == 0) { // Approximation does not know about the special 0 // But the iterator does.... @@ -489,10 +513,14 @@ PostingListAttributeTest::checkSearch(bool useBitVector, const AttributeVector & docBegin++; } TermFieldMatchData tfmd; + if (!need_unpack) { + tfmd.tagAsNotNeeded(); + } auto it = sc->createIterator(&tfmd, true); + EXPECT_EQ((useBitVector || !has_btree || !need_unpack) && has_bitvector, it->isBitVector()); it->initFullRange(); EXPECT_TRUE(it->seekFirst(docBegin)); - EXPECT_EQUAL(docBegin, it->getDocId()); + EXPECT_EQ(docBegin, it->getDocId()); size_t hits(0); uint32_t lastDocId = it->getDocId(); while (! it->isAtEnd()) { @@ -500,15 +528,22 @@ PostingListAttributeTest::checkSearch(bool useBitVector, const AttributeVector & it->seek(lastDocId+1); hits++; } - EXPECT_EQUAL(numHits, hits); - EXPECT_GREATER_EQUAL(approx, hits); - EXPECT_EQUAL(docEnd, lastDocId+1); + EXPECT_EQ(numHits, hits); + EXPECT_GE(approx, hits); + EXPECT_EQ(docEnd, lastDocId+1); +} + + +AttributePtr +create_attribute(const vespalib::stringref name, const Config& cfg) +{ + return AttributeFactory::createAttribute(tmp_dir + "/" + name, cfg); } AttributePtr create_as(const AttributeVector& attr, const std::string& name_suffix) { - return AttributeFactory::createAttribute(attr.getName() + name_suffix, attr.getConfig()); + return create_attribute(attr.getName() + name_suffix, attr.getConfig()); } template <typename VectorType, typename BufferType> @@ -559,19 +594,20 @@ PostingListAttributeTest::testPostingList(const AttributePtr& ptr1, uint32_t num void PostingListAttributeTest::testPostingList() { - testPostingList(false); - testPostingList(true); + testPostingList(false, false); + testPostingList(true, false); + testPostingList(true, true); } void -PostingListAttributeTest::testPostingList(bool enableBitVector) +PostingListAttributeTest::testPostingList(bool enableBitVector, bool enable_only_bitvector) { - testPostingList(enableBitVector, 1000, 50); - testPostingList(enableBitVector, 2000, 10); // This should force bitvector + testPostingList(enableBitVector, enable_only_bitvector, 1000, 50); + testPostingList(enableBitVector, enable_only_bitvector, 2000, 10); // This should force bitvector } void -PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs, uint32_t numUniqueValues) +PostingListAttributeTest::testPostingList(bool enableBitVector, bool enable_only_bitvector, uint32_t numDocs, uint32_t numUniqueValues) { { // IntegerAttribute @@ -583,21 +619,24 @@ PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("sint32", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("sint32", cfg); testPostingList<Int32PostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::ARRAY)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("aint32", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("aint32", cfg); testPostingList<Int32ArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::INT32, CollectionType::WSET)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsint32", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("wsint32", cfg); testPostingList<Int32WsetPostingListAttribute>(ptr1, numDocs, values); } } @@ -611,21 +650,24 @@ PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs Config cfg(Config(BasicType::FLOAT, CollectionType::SINGLE)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("sfloat", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("sfloat", cfg); testPostingList<FloatPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::ARRAY)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("afloat", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("afloat", cfg); testPostingList<FloatArrayPostingListAttribute>(ptr1, numDocs, values); } { Config cfg(Config(BasicType::FLOAT, CollectionType::WSET)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsfloat", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("wsfloat", cfg); testPostingList<FloatWsetPostingListAttribute>(ptr1, numDocs, values); } } @@ -645,21 +687,24 @@ PostingListAttributeTest::testPostingList(bool enableBitVector, uint32_t numDocs Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("sstr", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("sstr", cfg); testPostingList<StringPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::ARRAY)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("astr", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("astr", cfg); testPostingList<StringArrayPostingListAttribute>(ptr1, numDocs, charValues); } { Config cfg(Config(BasicType::STRING, CollectionType::WSET)); cfg.setFastSearch(true); cfg.setEnableBitVectors(enableBitVector); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsstr", cfg); + cfg.setEnableOnlyBitVector(enable_only_bitvector); + AttributePtr ptr1 = create_attribute("wsstr", cfg); testPostingList<StringWsetPostingListAttribute>(ptr1, numDocs, charValues); } } @@ -681,7 +726,7 @@ PostingListAttributeTest::checkPostingList(AttributeType & vec, ValueType value, DocSet::iterator docBegin = expected.begin(); DocSet::iterator docEnd = expected.end(); for (; postings.valid(); ++postings) { - EXPECT_EQUAL(*docBegin++, postings.getKey()); + EXPECT_EQ(*docBegin++, postings.getKey()); } EXPECT_TRUE(docBegin == docEnd); } @@ -789,14 +834,14 @@ PostingListAttributeTest::testArithmeticValueUpdate() { // IntegerAttribute Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); - AttributePtr ptr = AttributeFactory::createAttribute("sint32", cfg); + AttributePtr ptr = create_attribute("sint32", cfg); testArithmeticValueUpdate<Int32PostingListAttribute, largeint_t>(ptr); } { // FloatingPointAttribute Config cfg(Config(BasicType::FLOAT, CollectionType::SINGLE)); cfg.setFastSearch(true); - AttributePtr ptr = AttributeFactory::createAttribute("sfloat", cfg); + AttributePtr ptr = create_attribute("sfloat", cfg); testArithmeticValueUpdate<FloatPostingListAttribute, double>(ptr); } } @@ -823,7 +868,7 @@ PostingListAttributeTest::testReload(const AttributePtr & ptr1, const AttributeP ValueType buffer[1]; for (uint32_t doc = 0; doc < 5; ++doc) { EXPECT_TRUE(ptr2->get(doc, buffer, 1) == 1); - EXPECT_EQUAL(buffer[0], value); + EXPECT_EQ(buffer[0], value); } } @@ -834,13 +879,13 @@ PostingListAttributeTest::testReload() Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); { - AttributePtr ptr1 = AttributeFactory::createAttribute("sint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sint32_2", cfg); + AttributePtr ptr1 = create_attribute("sint32_1", cfg); + AttributePtr ptr2 = create_attribute("sint32_2", cfg); testReload<Int32PostingListAttribute, largeint_t>(ptr1, ptr2, 100); } { - AttributePtr ptr1 = AttributeFactory::createAttribute("sint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sint32_2", cfg); + AttributePtr ptr1 = create_attribute("sint32_1", cfg); + AttributePtr ptr2 = create_attribute("sint32_2", cfg); testReload<Int32PostingListAttribute, largeint_t>(ptr1, ptr2, 0); } } @@ -849,13 +894,13 @@ PostingListAttributeTest::testReload() Config cfg(Config(BasicType::FLOAT, CollectionType::SINGLE)); cfg.setFastSearch(true); { - AttributePtr ptr1 = AttributeFactory::createAttribute("sfloat_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sfloat_2", cfg); + AttributePtr ptr1 = create_attribute("sfloat_1", cfg); + AttributePtr ptr2 = create_attribute("sfloat_2", cfg); testReload<FloatPostingListAttribute, double>(ptr1, ptr2, 100); } { - AttributePtr ptr1 = AttributeFactory::createAttribute("sfloat_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sfloat_2", cfg); + AttributePtr ptr1 = create_attribute("sfloat_1", cfg); + AttributePtr ptr2 = create_attribute("sfloat_2", cfg); testReload<FloatPostingListAttribute, double>(ptr1, ptr2, 0); } } @@ -864,13 +909,13 @@ PostingListAttributeTest::testReload() Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); { - AttributePtr ptr1 = AttributeFactory::createAttribute("sstr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sstr_2", cfg); + AttributePtr ptr1 = create_attribute("sstr_1", cfg); + AttributePtr ptr2 = create_attribute("sstr_2", cfg); testReload<StringPostingListAttribute, vespalib::string>(ptr1, ptr2, "unique"); } { - AttributePtr ptr1 = AttributeFactory::createAttribute("sstr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sstr_2", cfg); + AttributePtr ptr1 = create_attribute("sstr_1", cfg); + AttributePtr ptr2 = create_attribute("sstr_2", cfg); testReload<StringPostingListAttribute, vespalib::string>(ptr1, ptr2, ""); } } @@ -893,18 +938,18 @@ PostingListAttributeTest::testMinMax(AttributePtr &ptr1, uint32_t trimmed) if (ptr1->hasMultiValue()) { if (trimmed == 2u) { - EXPECT_EQUAL(3, mmpi->getMinWeight()); + EXPECT_EQ(3, mmpi->getMinWeight()); } else { - EXPECT_EQUAL(-3, mmpi->getMinWeight()); + EXPECT_EQ(-3, mmpi->getMinWeight()); } - EXPECT_EQUAL(3, mmpi->getMaxWeight()); + EXPECT_EQ(3, mmpi->getMaxWeight()); } else { - EXPECT_EQUAL(1, mmpi->getMinWeight()); - EXPECT_EQUAL(1, mmpi->getMaxWeight()); + EXPECT_EQ(1, mmpi->getMinWeight()); + EXPECT_EQ(1, mmpi->getMaxWeight()); } sb->seek(1u); - EXPECT_EQUAL(1u, sb->getDocId()); + EXPECT_EQ(1u, sb->getDocId()); sc = getSearch2<VectorType>(as<VectorType>(ptr1)); sc->fetchPostings(queryeval::ExecuteInfo::TRUE); @@ -921,14 +966,14 @@ PostingListAttributeTest::testMinMax(AttributePtr &ptr1, uint32_t trimmed) if (ptr1->hasMultiValue()) { if (trimmed == 0) { - EXPECT_EQUAL(12, mmpi->getMinWeight()); + EXPECT_EQ(12, mmpi->getMinWeight()); } else { - EXPECT_EQUAL(14, mmpi->getMinWeight()); + EXPECT_EQ(14, mmpi->getMinWeight()); } - EXPECT_EQUAL(14, mmpi->getMaxWeight()); + EXPECT_EQ(14, mmpi->getMaxWeight()); } else { - EXPECT_EQUAL(1, mmpi->getMinWeight()); - EXPECT_EQUAL(1, mmpi->getMaxWeight()); + EXPECT_EQ(1, mmpi->getMinWeight()); + EXPECT_EQ(1, mmpi->getMaxWeight()); } } @@ -936,7 +981,7 @@ PostingListAttributeTest::testMinMax(AttributePtr &ptr1, uint32_t trimmed) if (trimmed == 2u) { EXPECT_TRUE(sb->isAtEnd()); } else { - EXPECT_EQUAL(7u, sb->getDocId()); + EXPECT_EQ(7u, sb->getDocId()); } } @@ -948,19 +993,29 @@ PostingListAttributeTest::testMinMax(AttributePtr &ptr1, AttributePtr &ptr2) addDocs(ptr1, numDocs); populate(as<VectorType>(ptr1)); - TEST_DO(testMinMax<VectorType>(ptr1, 0u)); + { + SCOPED_TRACE("after populate"); + testMinMax<VectorType>(ptr1, 0u); + } ASSERT_TRUE(ptr1->save(ptr2->getBaseFileName())); ASSERT_TRUE(ptr2->load()); - testMinMax<VectorType>(ptr2, 0u); + { + SCOPED_TRACE("after save and load"); + testMinMax<VectorType>(ptr2, 0u); + } ptr2->clearDoc(20); ptr2->clearDoc(25); ptr2->commit(); - TEST_DO(testMinMax<VectorType>(ptr2, 1u)); + { + SCOPED_TRACE("after 1 trim round"); + testMinMax<VectorType>(ptr2, 1u); + } ptr2->clearDoc(7); ptr2->commit(); - TEST_DO(testMinMax<VectorType>(ptr2, 2u)); + SCOPED_TRACE("after 2 trim rounds"); + testMinMax<VectorType>(ptr2, 2u); } @@ -970,29 +1025,29 @@ PostingListAttributeTest::testMinMax() { Config cfg(Config(BasicType::INT32, CollectionType::SINGLE)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("sint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sint32_2", cfg); + AttributePtr ptr1 = create_attribute("sint32_1", cfg); + AttributePtr ptr2 = create_attribute("sint32_2", cfg); testMinMax<IntegerAttribute>(ptr1, ptr2); } { Config cfg(Config(BasicType::INT32, CollectionType::WSET)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsint32_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("wsint32_2", cfg); + AttributePtr ptr1 = create_attribute("wsint32_1", cfg); + AttributePtr ptr2 = create_attribute("wsint32_2", cfg); testMinMax<IntegerAttribute>(ptr1, ptr2); } { Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("sstr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("sstr_2", cfg); + AttributePtr ptr1 = create_attribute("sstr_1", cfg); + AttributePtr ptr2 = create_attribute("sstr_2", cfg); testMinMax<StringAttribute>(ptr1, ptr2); } { Config cfg(Config(BasicType::STRING, CollectionType::WSET)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("wsstr_1", cfg); - AttributePtr ptr2 = AttributeFactory::createAttribute("wsstr_2", cfg); + AttributePtr ptr1 = create_attribute("wsstr_1", cfg); + AttributePtr ptr2 = create_attribute("wsstr_2", cfg); testMinMax<StringAttribute>(ptr1, ptr2); } } @@ -1003,7 +1058,7 @@ PostingListAttributeTest::testStringFold() { Config cfg(Config(BasicType::STRING, CollectionType::SINGLE)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("sstr_1", cfg); + AttributePtr ptr1 = create_attribute("sstr_1", cfg); addDocs(ptr1, 6); @@ -1046,7 +1101,7 @@ PostingListAttributeTest::testDupValuesInIntArray() { Config cfg(Config(BasicType::INT32, CollectionType::ARRAY)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("aint32_3", cfg); + AttributePtr ptr1 = create_attribute("aint32_3", cfg); addDocs(ptr1, 6); IntegerAttribute &ia(asInt(ptr1)); @@ -1071,7 +1126,7 @@ PostingListAttributeTest::testDupValuesInStringArray() { Config cfg(Config(BasicType::STRING, CollectionType::ARRAY)); cfg.setFastSearch(true); - AttributePtr ptr1 = AttributeFactory::createAttribute("astr_3", cfg); + AttributePtr ptr1 = create_attribute("astr_3", cfg); addDocs(ptr1, 6); StringAttribute &sa(asString(ptr1)); @@ -1102,22 +1157,50 @@ PostingListAttributeTest::testDupValuesInStringArray() } -int -PostingListAttributeTest::Main() +TEST_F(PostingListAttributeTest, test_posting_list) { - TEST_INIT("postinglistattribute_test"); - testPostingList(); +} + +TEST_F(PostingListAttributeTest, test_arithmetic_value_update) +{ testArithmeticValueUpdate(); +} + +TEST_F(PostingListAttributeTest, test_reload) +{ testReload(); +} + +TEST_F(PostingListAttributeTest, test_min_max) +{ testMinMax(); +} + +TEST_F(PostingListAttributeTest, test_string_fold) +{ testStringFold(); +} + +TEST_F(PostingListAttributeTest, test_dup_values_in_int_array) +{ testDupValuesInIntArray(); - testDupValuesInStringArray(); +} - TEST_DONE(); +TEST_F(PostingListAttributeTest, test_dup_values_in_string_array) +{ + testDupValuesInStringArray(); } } -TEST_APPHOOK(search::PostingListAttributeTest); +int +main(int argc, char* argv[]) +{ + ::testing::InitGoogleTest(&argc, argv); + vespalib::rmdir(tmp_dir, true); + vespalib::mkdir(tmp_dir, true); + int retval = RUN_ALL_TESTS(); + vespalib::rmdir(tmp_dir, true); + return retval; +} diff --git a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.sh b/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.sh deleted file mode 100755 index f5935f62a50..00000000000 --- a/searchlib/src/tests/attribute/postinglistattribute/postinglistattribute_test.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -$VALGRIND ./searchlib_postinglistattribute_test_app -rm -rf *.dat -rm -rf *.udat -rm -rf *.idx -rm -rf *.weight diff --git a/searchlib/src/tests/common/bitvector/bitvector_test.cpp b/searchlib/src/tests/common/bitvector/bitvector_test.cpp index a9850756ac6..79af28d20be 100644 --- a/searchlib/src/tests/common/bitvector/bitvector_test.cpp +++ b/searchlib/src/tests/common/bitvector/bitvector_test.cpp @@ -604,55 +604,55 @@ TEST("requireThatGrowWorks") { vespalib::GenerationHolder g; GrowableBitVector v(200, 200, g); - EXPECT_EQUAL(0u, v.countTrueBits()); + EXPECT_EQUAL(0u, v.writer().countTrueBits()); - v.setBitAndMaintainCount(7); - v.setBitAndMaintainCount(39); - v.setBitAndMaintainCount(71); - v.setBitAndMaintainCount(103); - EXPECT_EQUAL(4u, v.countTrueBits()); - - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(1023u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + v.writer().setBitAndMaintainCount(7); + v.writer().setBitAndMaintainCount(39); + v.writer().setBitAndMaintainCount(71); + v.writer().setBitAndMaintainCount(103); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); + + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(1023u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_TRUE(v.reserve(1024)); - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.extend(202)); - EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(202u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.shrink(200)); - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.reserve(2047)); - EXPECT_EQUAL(200u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(200u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.shrink(202)); - EXPECT_EQUAL(202u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71,103]", v)); - EXPECT_EQUAL(4u, v.countTrueBits()); + EXPECT_EQUAL(202u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71,103]", v.reader())); + EXPECT_EQUAL(4u, v.writer().countTrueBits()); EXPECT_FALSE(v.shrink(100)); - EXPECT_EQUAL(100u, v.size()); - EXPECT_EQUAL(2047u, v.capacity()); - EXPECT_TRUE(assertBV("[7,39,71]", v)); - EXPECT_EQUAL(3u, v.countTrueBits()); + EXPECT_EQUAL(100u, v.reader().size()); + EXPECT_EQUAL(2047u, v.writer().capacity()); + EXPECT_TRUE(assertBV("[7,39,71]", v.reader())); + EXPECT_EQUAL(3u, v.writer().countTrueBits()); - v.invalidateCachedCount(); + v.writer().invalidateCachedCount(); EXPECT_TRUE(v.reserve(3100)); - EXPECT_EQUAL(100u, v.size()); - EXPECT_EQUAL(4095u, v.capacity()); - EXPECT_EQUAL(3u, v.countTrueBits()); + EXPECT_EQUAL(100u, v.reader().size()); + EXPECT_EQUAL(4095u, v.writer().capacity()); + EXPECT_EQUAL(3u, v.writer().countTrueBits()); g.transferHoldLists(1); g.trimHoldLists(2); @@ -666,7 +666,7 @@ TEST("require that growable bit vectors keeps memory allocator") vespalib::GenerationHolder g; GrowableBitVector v(200, 200, g, &init_alloc); EXPECT_EQUAL(AllocStats(1, 0), stats); - v.resize(1); + v.writer().resize(1); // DO NOT TRY THIS AT HOME EXPECT_EQUAL(AllocStats(2, 1), stats); v.reserve(2000); EXPECT_EQUAL(AllocStats(3, 1), stats); @@ -674,7 +674,7 @@ TEST("require that growable bit vectors keeps memory allocator") EXPECT_EQUAL(AllocStats(4, 1), stats); v.shrink(200); EXPECT_EQUAL(AllocStats(4, 1), stats); - v.resize(1); + v.writer().resize(1); // DO NOT TRY THIS AT HOME EXPECT_EQUAL(AllocStats(5, 2), stats); g.transferHoldLists(1); g.trimHoldLists(2); diff --git a/searchlib/src/tests/features/tensor/tensor_test.cpp b/searchlib/src/tests/features/tensor/tensor_test.cpp index 1f9db526ec4..a8c4d6714d8 100644 --- a/searchlib/src/tests/features/tensor/tensor_test.cpp +++ b/searchlib/src/tests/features/tensor/tensor_test.cpp @@ -87,6 +87,10 @@ struct ExecFixture void setQueryTensorType(const vespalib::string &queryFeatureName, const vespalib::string &type) { type::QueryFeature::set(test.getIndexEnv().getProperties(), queryFeatureName, type); } + void setQueryTensorDefault(const vespalib::string &tensorName, const vespalib::string &expr) { + vespalib::string key = "query(" + tensorName + ")"; + test.getIndexEnv().getProperties().add(key, expr); + } void setupAttributeVectors() { std::vector<AttributePtr> attrs; attrs.push_back(createTensorAttribute("tensorattr", "tensor(x{})")); @@ -149,6 +153,8 @@ struct ExecFixture .add({{"x", "0"},{"y", "1"}}, 13 ) .add({{"x", "1"},{"y", "0"}}, 17 ))); setQueryTensorType("null", "tensor(q{})"); + setQueryTensorType("with_default", "tensor(x[3])"); + setQueryTensorDefault("with_default", "tensor(x[3])(x+1)"); } const Value &extractTensor(uint32_t docid) { Value::CREF value = test.resolveObjectFeature(docid); @@ -187,6 +193,15 @@ TEST_F("require that tensor from query can be extracted as tensor in query featu .add({{"q", "e"}}, 13), spec_from_value(f.execute())); } +TEST_F("require that tensor from query can have default value", + ExecFixture("query(with_default)")) +{ + EXPECT_EQUAL(TensorSpec("tensor(x[3])") + .add({{"x", 0}}, 1) + .add({{"x", 1}}, 2) + .add({{"x", 2}}, 3), spec_from_value(f.execute())); +} + TEST_F("require that empty tensor is created if attribute does not exists", ExecFixture("attribute(null)")) { diff --git a/searchlib/src/tests/forcelink/CMakeLists.txt b/searchlib/src/tests/forcelink/CMakeLists.txt index 81ce6beab32..a6af5a77b0c 100644 --- a/searchlib/src/tests/forcelink/CMakeLists.txt +++ b/searchlib/src/tests/forcelink/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(searchlib_forcelink_test_app TEST forcelink.cpp DEPENDS searchlib - searchlib_searchlib_uca ) vespa_add_test(NAME searchlib_forcelink_test_app COMMAND searchlib_forcelink_test_app) diff --git a/searchcommon/src/tests/.gitignore b/searchlib/src/tests/searchcommon/.gitignore index a3e9c375723..a3e9c375723 100644 --- a/searchcommon/src/tests/.gitignore +++ b/searchlib/src/tests/searchcommon/.gitignore diff --git a/searchcommon/src/tests/attribute/config/.gitignore b/searchlib/src/tests/searchcommon/attribute/config/.gitignore index ffdb7b1e933..ffdb7b1e933 100644 --- a/searchcommon/src/tests/attribute/config/.gitignore +++ b/searchlib/src/tests/searchcommon/attribute/config/.gitignore diff --git a/searchcommon/src/tests/attribute/config/CMakeLists.txt b/searchlib/src/tests/searchcommon/attribute/config/CMakeLists.txt index 71df8414aee..f61138c5d73 100644 --- a/searchcommon/src/tests/attribute/config/CMakeLists.txt +++ b/searchlib/src/tests/searchcommon/attribute/config/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(searchcommon_attribute_config_test_app TEST SOURCES attribute_config_test.cpp DEPENDS - searchcommon + searchlib ) vespa_add_test(NAME searchcommon_attribute_config_test_app NO_VALGRIND COMMAND searchcommon_attribute_config_test_app) diff --git a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp index 918e14546e6..918e14546e6 100644 --- a/searchcommon/src/tests/attribute/config/attribute_config_test.cpp +++ b/searchlib/src/tests/searchcommon/attribute/config/attribute_config_test.cpp diff --git a/searchcommon/src/tests/schema/.gitignore b/searchlib/src/tests/searchcommon/schema/.gitignore index e000f0ca2c8..e000f0ca2c8 100644 --- a/searchcommon/src/tests/schema/.gitignore +++ b/searchlib/src/tests/searchcommon/schema/.gitignore diff --git a/searchcommon/src/tests/schema/CMakeLists.txt b/searchlib/src/tests/searchcommon/schema/CMakeLists.txt index 515a5206f04..2304c319dea 100644 --- a/searchcommon/src/tests/schema/CMakeLists.txt +++ b/searchlib/src/tests/searchcommon/schema/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(searchcommon_schema_test_app TEST SOURCES schema_test.cpp DEPENDS - searchcommon + searchlib GTest::GTest ) vespa_add_test(NAME searchcommon_schema_test_app NO_VALGRIND COMMAND searchcommon_schema_test_app) diff --git a/searchcommon/src/tests/schema/imported-fields-cfg/attributes.cfg b/searchlib/src/tests/searchcommon/schema/imported-fields-cfg/attributes.cfg index 9a08f7e2324..9a08f7e2324 100644 --- a/searchcommon/src/tests/schema/imported-fields-cfg/attributes.cfg +++ b/searchlib/src/tests/searchcommon/schema/imported-fields-cfg/attributes.cfg diff --git a/searchcommon/src/tests/schema/load-save-cfg/attributes.cfg b/searchlib/src/tests/searchcommon/schema/load-save-cfg/attributes.cfg index 09f711b6a65..09f711b6a65 100644 --- a/searchcommon/src/tests/schema/load-save-cfg/attributes.cfg +++ b/searchlib/src/tests/searchcommon/schema/load-save-cfg/attributes.cfg diff --git a/searchcommon/src/tests/schema/load-save-cfg/indexschema.cfg b/searchlib/src/tests/searchcommon/schema/load-save-cfg/indexschema.cfg index b9d82b9b569..b9d82b9b569 100644 --- a/searchcommon/src/tests/schema/load-save-cfg/indexschema.cfg +++ b/searchlib/src/tests/searchcommon/schema/load-save-cfg/indexschema.cfg diff --git a/searchcommon/src/tests/schema/load-save-cfg/summary.cfg b/searchlib/src/tests/searchcommon/schema/load-save-cfg/summary.cfg index 0c2de33d076..0c2de33d076 100644 --- a/searchcommon/src/tests/schema/load-save-cfg/summary.cfg +++ b/searchlib/src/tests/searchcommon/schema/load-save-cfg/summary.cfg diff --git a/searchcommon/src/tests/schema/schema-without-index-field-properties.txt b/searchlib/src/tests/searchcommon/schema/schema-without-index-field-properties.txt index 4491b1242e0..4491b1242e0 100644 --- a/searchcommon/src/tests/schema/schema-without-index-field-properties.txt +++ b/searchlib/src/tests/searchcommon/schema/schema-without-index-field-properties.txt diff --git a/searchcommon/src/tests/schema/schema_test.cpp b/searchlib/src/tests/searchcommon/schema/schema_test.cpp index 09a7359bac7..09a7359bac7 100644 --- a/searchcommon/src/tests/schema/schema_test.cpp +++ b/searchlib/src/tests/searchcommon/schema/schema_test.cpp diff --git a/searchlib/src/tests/sort/CMakeLists.txt b/searchlib/src/tests/sort/CMakeLists.txt index c59b3a4d8af..e342237adf0 100644 --- a/searchlib/src/tests/sort/CMakeLists.txt +++ b/searchlib/src/tests/sort/CMakeLists.txt @@ -11,7 +11,6 @@ vespa_add_executable(searchlib_sort_test_app sort_test.cpp DEPENDS searchlib - searchlib_searchlib_uca ) #vespa_add_test(NAME searchlib_sort_test_app COMMAND searchlib_sort_test_app) vespa_add_executable(searchlib_uca_stress_app @@ -19,6 +18,5 @@ vespa_add_executable(searchlib_uca_stress_app uca.cpp DEPENDS searchlib - searchlib_searchlib_uca ) vespa_add_test(NAME searchlib_uca_stress_app COMMAND searchlib_uca_stress_app BENCHMARK) diff --git a/searchlib/src/tests/sortspec/CMakeLists.txt b/searchlib/src/tests/sortspec/CMakeLists.txt index 5ebaa22f154..a9b3a906804 100644 --- a/searchlib/src/tests/sortspec/CMakeLists.txt +++ b/searchlib/src/tests/sortspec/CMakeLists.txt @@ -4,6 +4,5 @@ vespa_add_executable(searchlib_multilevelsort_test_app TEST multilevelsort.cpp DEPENDS searchlib - searchlib_searchlib_uca ) vespa_add_test(NAME searchlib_multilevelsort_test_app COMMAND searchlib_multilevelsort_test_app) diff --git a/searchcommon/src/vespa/searchcommon/.gitignore b/searchlib/src/vespa/searchcommon/.gitignore index f76a9d84bed..f76a9d84bed 100644 --- a/searchcommon/src/vespa/searchcommon/.gitignore +++ b/searchlib/src/vespa/searchcommon/.gitignore diff --git a/searchcommon/src/vespa/searchcommon/common/.gitignore b/searchlib/src/vespa/searchcommon/attribute/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/searchcommon/src/vespa/searchcommon/common/.gitignore +++ b/searchlib/src/vespa/searchcommon/attribute/.gitignore diff --git a/searchcommon/src/vespa/searchcommon/attribute/CMakeLists.txt b/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt index 704fe238ed5..704fe238ed5 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/CMakeLists.txt +++ b/searchlib/src/vespa/searchcommon/attribute/CMakeLists.txt diff --git a/searchcommon/src/vespa/searchcommon/attribute/attribute_utils.cpp b/searchlib/src/vespa/searchcommon/attribute/attribute_utils.cpp index cd5cc58c75c..cd5cc58c75c 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/attribute_utils.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/attribute_utils.cpp diff --git a/searchcommon/src/vespa/searchcommon/attribute/attribute_utils.h b/searchlib/src/vespa/searchcommon/attribute/attribute_utils.h index e4c2a8e4727..e4c2a8e4727 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/attribute_utils.h +++ b/searchlib/src/vespa/searchcommon/attribute/attribute_utils.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h b/searchlib/src/vespa/searchcommon/attribute/attributecontent.h index f5960ce358b..f5960ce358b 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/attributecontent.h +++ b/searchlib/src/vespa/searchcommon/attribute/attributecontent.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp b/searchlib/src/vespa/searchcommon/attribute/basictype.cpp index 5bab2fc06d2..5bab2fc06d2 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/basictype.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.cpp diff --git a/searchcommon/src/vespa/searchcommon/attribute/basictype.h b/searchlib/src/vespa/searchcommon/attribute/basictype.h index bd7b4a2b4bc..bd7b4a2b4bc 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/basictype.h +++ b/searchlib/src/vespa/searchcommon/attribute/basictype.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp b/searchlib/src/vespa/searchcommon/attribute/collectiontype.cpp index b77382f6126..b77382f6126 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/collectiontype.cpp diff --git a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h b/searchlib/src/vespa/searchcommon/attribute/collectiontype.h index 35cb7612ed0..35cb7612ed0 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/collectiontype.h +++ b/searchlib/src/vespa/searchcommon/attribute/collectiontype.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.cpp b/searchlib/src/vespa/searchcommon/attribute/config.cpp index 0a50faa04c0..0a50faa04c0 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/config.cpp diff --git a/searchcommon/src/vespa/searchcommon/attribute/config.h b/searchlib/src/vespa/searchcommon/attribute/config.h index f572f5038fc..f572f5038fc 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/config.h +++ b/searchlib/src/vespa/searchcommon/attribute/config.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h b/searchlib/src/vespa/searchcommon/attribute/distance_metric.h index 26efa30bba4..26efa30bba4 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/distance_metric.h +++ b/searchlib/src/vespa/searchcommon/attribute/distance_metric.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h b/searchlib/src/vespa/searchcommon/attribute/hnsw_index_params.h index 4f9d3c5593c..4f9d3c5593c 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/hnsw_index_params.h +++ b/searchlib/src/vespa/searchcommon/attribute/hnsw_index_params.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h b/searchlib/src/vespa/searchcommon/attribute/i_attribute_functor.h index da5127de8ee..da5127de8ee 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_attribute_functor.h +++ b/searchlib/src/vespa/searchcommon/attribute/i_attribute_functor.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h b/searchlib/src/vespa/searchcommon/attribute/i_multi_value_attribute.h index ea1fbe0b2b4..ea1fbe0b2b4 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_attribute.h +++ b/searchlib/src/vespa/searchcommon/attribute/i_multi_value_attribute.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_read_view.h b/searchlib/src/vespa/searchcommon/attribute/i_multi_value_read_view.h index 8e5005eae8d..8e5005eae8d 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_multi_value_read_view.h +++ b/searchlib/src/vespa/searchcommon/attribute/i_multi_value_read_view.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h b/searchlib/src/vespa/searchcommon/attribute/i_search_context.h index ff62c535e7f..ff62c535e7f 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/i_search_context.h +++ b/searchlib/src/vespa/searchcommon/attribute/i_search_context.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h b/searchlib/src/vespa/searchcommon/attribute/iattributecontext.h index bb349057ca9..bb349057ca9 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributecontext.h +++ b/searchlib/src/vespa/searchcommon/attribute/iattributecontext.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h index fa91f301b92..fa91f301b92 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/iattributevector.h +++ b/searchlib/src/vespa/searchcommon/attribute/iattributevector.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/multi_value_traits.h b/searchlib/src/vespa/searchcommon/attribute/multi_value_traits.h index f03b031f991..f03b031f991 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/multi_value_traits.h +++ b/searchlib/src/vespa/searchcommon/attribute/multi_value_traits.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/multivalue.h b/searchlib/src/vespa/searchcommon/attribute/multivalue.h index 2ed8309188e..2ed8309188e 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/multivalue.h +++ b/searchlib/src/vespa/searchcommon/attribute/multivalue.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h b/searchlib/src/vespa/searchcommon/attribute/persistent_predicate_params.h index d81eb9c5d3c..d81eb9c5d3c 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/persistent_predicate_params.h +++ b/searchlib/src/vespa/searchcommon/attribute/persistent_predicate_params.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h b/searchlib/src/vespa/searchcommon/attribute/predicate_params.h index 133b7331689..133b7331689 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/predicate_params.h +++ b/searchlib/src/vespa/searchcommon/attribute/predicate_params.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp b/searchlib/src/vespa/searchcommon/attribute/search_context_params.cpp index 2e8aba6f5f8..2e8aba6f5f8 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/search_context_params.cpp diff --git a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h b/searchlib/src/vespa/searchcommon/attribute/search_context_params.h index 168f4215ef6..168f4215ef6 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/search_context_params.h +++ b/searchlib/src/vespa/searchcommon/attribute/search_context_params.h diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.cpp b/searchlib/src/vespa/searchcommon/attribute/status.cpp index a7d1f5b3d38..a7d1f5b3d38 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.cpp +++ b/searchlib/src/vespa/searchcommon/attribute/status.cpp diff --git a/searchcommon/src/vespa/searchcommon/attribute/status.h b/searchlib/src/vespa/searchcommon/attribute/status.h index f2212d4c76a..f2212d4c76a 100644 --- a/searchcommon/src/vespa/searchcommon/attribute/status.h +++ b/searchlib/src/vespa/searchcommon/attribute/status.h diff --git a/searchcommon/src/vespa/searchcommon/config/.gitignore b/searchlib/src/vespa/searchcommon/common/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/searchcommon/src/vespa/searchcommon/config/.gitignore +++ b/searchlib/src/vespa/searchcommon/common/.gitignore diff --git a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt b/searchlib/src/vespa/searchcommon/common/CMakeLists.txt index 6cc02ae7884..6cc02ae7884 100644 --- a/searchcommon/src/vespa/searchcommon/common/CMakeLists.txt +++ b/searchlib/src/vespa/searchcommon/common/CMakeLists.txt diff --git a/searchcommon/src/vespa/searchcommon/common/datatype.cpp b/searchlib/src/vespa/searchcommon/common/datatype.cpp index 1fe3a488aac..1fe3a488aac 100644 --- a/searchcommon/src/vespa/searchcommon/common/datatype.cpp +++ b/searchlib/src/vespa/searchcommon/common/datatype.cpp diff --git a/searchcommon/src/vespa/searchcommon/common/datatype.h b/searchlib/src/vespa/searchcommon/common/datatype.h index e1c6a44b620..e1c6a44b620 100644 --- a/searchcommon/src/vespa/searchcommon/common/datatype.h +++ b/searchlib/src/vespa/searchcommon/common/datatype.h diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp b/searchlib/src/vespa/searchcommon/common/dictionary_config.cpp index e1b990e5660..e1b990e5660 100644 --- a/searchcommon/src/vespa/searchcommon/common/dictionary_config.cpp +++ b/searchlib/src/vespa/searchcommon/common/dictionary_config.cpp diff --git a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h b/searchlib/src/vespa/searchcommon/common/dictionary_config.h index f51341ad799..f51341ad799 100644 --- a/searchcommon/src/vespa/searchcommon/common/dictionary_config.h +++ b/searchlib/src/vespa/searchcommon/common/dictionary_config.h diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp b/searchlib/src/vespa/searchcommon/common/growstrategy.cpp index f35cdbaa640..f35cdbaa640 100644 --- a/searchcommon/src/vespa/searchcommon/common/growstrategy.cpp +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.cpp diff --git a/searchcommon/src/vespa/searchcommon/common/growstrategy.h b/searchlib/src/vespa/searchcommon/common/growstrategy.h index b9b4a42cf72..b9b4a42cf72 100644 --- a/searchcommon/src/vespa/searchcommon/common/growstrategy.h +++ b/searchlib/src/vespa/searchcommon/common/growstrategy.h diff --git a/searchcommon/src/vespa/searchcommon/common/iblobconverter.h b/searchlib/src/vespa/searchcommon/common/iblobconverter.h index 6581c3e5ccb..6581c3e5ccb 100644 --- a/searchcommon/src/vespa/searchcommon/common/iblobconverter.h +++ b/searchlib/src/vespa/searchcommon/common/iblobconverter.h diff --git a/searchcommon/src/vespa/searchcommon/common/range.h b/searchlib/src/vespa/searchcommon/common/range.h index ea2553c129b..ea2553c129b 100644 --- a/searchcommon/src/vespa/searchcommon/common/range.h +++ b/searchlib/src/vespa/searchcommon/common/range.h diff --git a/searchcommon/src/vespa/searchcommon/common/schema.cpp b/searchlib/src/vespa/searchcommon/common/schema.cpp index c6a2a4532a3..c6a2a4532a3 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.cpp +++ b/searchlib/src/vespa/searchcommon/common/schema.cpp diff --git a/searchcommon/src/vespa/searchcommon/common/schema.h b/searchlib/src/vespa/searchcommon/common/schema.h index 3a9bcbdd904..3a9bcbdd904 100644 --- a/searchcommon/src/vespa/searchcommon/common/schema.h +++ b/searchlib/src/vespa/searchcommon/common/schema.h diff --git a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.cpp index ff69ca9818a..8fbebe80b4b 100644 --- a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.cpp +++ b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.cpp @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "schemaconfigurer.h" +#include "subscriptionproxyng.h" #include <vespa/config-attributes.h> #include <vespa/config-imported-fields.h> #include <vespa/config-indexschema.h> @@ -9,8 +10,6 @@ #include <vespa/searchcommon/attribute/collectiontype.h> #include <vespa/searchcommon/attribute/basictype.h> -#include <vespa/searchcommon/config/subscriptionproxyng.h> - #include <vespa/log/log.h> LOG_SETUP(".index.schemaconfigurer"); diff --git a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.h b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.h index 925aefcfa25..925aefcfa25 100644 --- a/searchcommon/src/vespa/searchcommon/common/schemaconfigurer.h +++ b/searchlib/src/vespa/searchcommon/common/schemaconfigurer.h diff --git a/searchcommon/src/vespa/searchcommon/config/subscriptionproxyng.h b/searchlib/src/vespa/searchcommon/common/subscriptionproxyng.h index dd24480f689..dd24480f689 100644 --- a/searchcommon/src/vespa/searchcommon/config/subscriptionproxyng.h +++ b/searchlib/src/vespa/searchcommon/common/subscriptionproxyng.h diff --git a/searchcommon/src/vespa/searchcommon/common/undefinedvalues.h b/searchlib/src/vespa/searchcommon/common/undefinedvalues.h index bbe3198a8dc..bbe3198a8dc 100644 --- a/searchcommon/src/vespa/searchcommon/common/undefinedvalues.h +++ b/searchlib/src/vespa/searchcommon/common/undefinedvalues.h diff --git a/searchlib/src/vespa/searchlib/CMakeLists.txt b/searchlib/src/vespa/searchlib/CMakeLists.txt index dac40e0ab5f..71442e27592 100644 --- a/searchlib/src/vespa/searchlib/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/CMakeLists.txt @@ -1,4 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +find_package(ICU 60.0 REQUIRED COMPONENTS uc i18n) vespa_add_library(searchlib SOURCES $<TARGET_OBJECTS:searchlib_aggregation> @@ -28,11 +29,16 @@ vespa_add_library(searchlib $<TARGET_OBJECTS:searchlib_searchlib_index> $<TARGET_OBJECTS:searchlib_tensor> $<TARGET_OBJECTS:searchlib_transactionlog> + $<TARGET_OBJECTS:searchlib_uca> $<TARGET_OBJECTS:searchlib_util> + $<TARGET_OBJECTS:searchcommon_searchcommon_common> + $<TARGET_OBJECTS:searchcommon_searchcommon_attribute> INSTALL lib64 DEPENDS - staging_vespalib + vespalib + ICU::i18n + ICU::uc ${VESPA_ATOMIC_LIB} ) diff --git a/searchlib/src/vespa/searchlib/aggregation/grouping.cpp b/searchlib/src/vespa/searchlib/aggregation/grouping.cpp index f373b5fc0b3..c07e399b38e 100644 --- a/searchlib/src/vespa/searchlib/aggregation/grouping.cpp +++ b/searchlib/src/vespa/searchlib/aggregation/grouping.cpp @@ -121,8 +121,7 @@ Grouping::Grouping() Grouping::Grouping(const Grouping &) = default; Grouping & Grouping::operator = (const Grouping &) = default; - -Grouping::~Grouping() { } +Grouping::~Grouping() = default; void Grouping::selectMembers(const vespalib::ObjectPredicate &predicate, diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp index d6aa136e15f..6e19d35a966 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.cpp @@ -62,7 +62,7 @@ void FlagAttributeT<B>::clearOldValues(DocId doc) { const typename B::WType * values(nullptr); for (uint32_t i(0), m(this->get(doc, values)); i < m; i++) { - BitVector * bv = _bitVectors[getOffset(multivalue::get_value(values[i]))]; + BitVector * bv = _bitVectors[getOffset(multivalue::get_value(values[i]))].load_relaxed(); if (bv != nullptr) { bv->clearBitAndMaintainCount(doc); } @@ -103,7 +103,7 @@ bool FlagAttributeT<B>::onLoad(vespalib::Executor * executor) { for (size_t i(0), m(_bitVectors.size()); i < m; i++) { _bitVectorStore[i].reset(); - _bitVectors[i] = nullptr; + _bitVectors[i].store_relaxed(nullptr); } _bitVectorSize = 0; return B::onLoad(executor); @@ -119,12 +119,12 @@ void FlagAttributeT<B>::setNewValues(DocId doc, const std::vector<typename B::WT for (uint32_t i(0), m(values.size()); i < m; i++) { typename B::WType value = values[i]; uint32_t offset = getOffset(value); - BitVector * bv = _bitVectors[offset]; + BitVector * bv = _bitVectors[offset].load_relaxed(); if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); - _bitVectors[offset] = _bitVectorStore[offset].get(); - bv = _bitVectors[offset]; + _bitVectors[offset].store_release(&_bitVectorStore[offset]->writer()); + bv = _bitVectors[offset].load_relaxed(); ensureGuardBit(*bv); } bv->setBitAndMaintainCount(doc); @@ -136,12 +136,12 @@ void FlagAttributeT<B>::setNewBVValue(DocId doc, multivalue::ValueType_t<typename B::WType> value) { uint32_t offset = getOffset(value); - BitVector * bv = _bitVectors[offset]; + BitVector * bv = _bitVectors[offset].load_relaxed(); if (bv == nullptr) { assert(_bitVectorSize >= this->getNumDocs()); _bitVectorStore[offset] = std::make_shared<GrowableBitVector>(_bitVectorSize, _bitVectorSize, _bitVectorHolder); - _bitVectors[offset] = _bitVectorStore[offset].get(); - bv = _bitVectors[offset]; + _bitVectors[offset].store_release(&_bitVectorStore[offset]->writer()); + bv = _bitVectors[offset].load_relaxed(); ensureGuardBit(*bv); } bv->setBitAndMaintainCount(doc); @@ -186,7 +186,8 @@ template <typename B> void FlagAttributeT<B>::ensureGuardBit() { - for (BitVector * bv : _bitVectors) { + for (const auto &wrapper: _bitVectors) { + BitVector *bv = wrapper.load_relaxed(); if (bv != nullptr) { ensureGuardBit(*bv); } @@ -197,7 +198,8 @@ template <typename B> void FlagAttributeT<B>::clearGuardBit(DocId doc) { - for (BitVector * bv : _bitVectors) { + for (const auto &wrapper: _bitVectors) { + BitVector *bv = wrapper.load_relaxed(); if (bv != nullptr) { bv->clearBit(doc); // clear guard bit and start using this doc id } @@ -211,9 +213,12 @@ FlagAttributeT<B>::resizeBitVectors(uint32_t neededSize) const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = neededSize + (neededSize * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); for (size_t i(0), m(_bitVectors.size()); i < m; i++) { - BitVector *bv = _bitVectors[i]; + BitVector *bv = _bitVectors[i].load_relaxed(); if (bv != nullptr) { - _bitVectorStore[i]->extend(newSize); + if (_bitVectorStore[i]->extend(newSize)) { + _bitVectors[i].store_release(&_bitVectorStore[i]->writer()); + bv = _bitVectors[i].load_relaxed(); + } ensureGuardBit(*bv); } } diff --git a/searchlib/src/vespa/searchlib/attribute/flagattribute.h b/searchlib/src/vespa/searchlib/attribute/flagattribute.h index c4f4980e372..796c1493cc9 100644 --- a/searchlib/src/vespa/searchlib/attribute/flagattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/flagattribute.h @@ -4,6 +4,7 @@ #include "multinumericattribute.h" #include "multi_numeric_search_context.h" #include <vespa/searchlib/common/growablebitvector.h> +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> namespace search { @@ -35,9 +36,10 @@ private: void removeOldGenerations(vespalib::GenerationHandler::generation_t firstUsed) override; uint32_t getOffset(int8_t value) const { return value + 128; } + using AtomicBitVectorPtr = vespalib::datastore::AtomicValueWrapper<BitVector *>; vespalib::GenerationHolder _bitVectorHolder; std::vector<std::shared_ptr<GrowableBitVector> > _bitVectorStore; - std::vector<BitVector *> _bitVectors; + std::vector<AtomicBitVectorPtr> _bitVectors; uint32_t _bitVectorSize; }; diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp index 6a0be483a55..045d80bc6c4 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.cpp @@ -13,7 +13,7 @@ namespace search::attribute { using queryeval::SearchIterator; template <typename T, typename M> -MultiNumericFlagSearchContext<T, M>::MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors) +MultiNumericFlagSearchContext<T, M>::MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, AtomicBitVectorsRef bit_vectors) : MultiNumericSearchContext<T, M>(std::move(qTerm), toBeSearched, mv_mapping_read_view), _bit_vectors(bit_vectors), _zeroHits(false) diff --git a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h index 00f7077b2d0..dd88ee4202c 100644 --- a/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h +++ b/searchlib/src/vespa/searchlib/attribute/multi_numeric_flag_search_context.h @@ -3,6 +3,7 @@ #pragma once #include "multi_numeric_search_context.h" +#include <vespa/vespalib/datastore/atomic_value_wrapper.h> namespace search { class BitVector; @@ -20,16 +21,19 @@ template <typename T, typename M> class MultiNumericFlagSearchContext : public MultiNumericSearchContext<T, M> { public: - MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, vespalib::ConstArrayRef<BitVector *> bit_vectors); + using AtomicBitVectorsRef = vespalib::ConstArrayRef<vespalib::datastore::AtomicValueWrapper<BitVector *>>; + + MultiNumericFlagSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const AttributeVector& toBeSearched, MultiValueMappingReadView<M> mv_mapping_read_view, + AtomicBitVectorsRef bit_vectors); std::unique_ptr<queryeval::SearchIterator> createIterator(fef::TermFieldMatchData * matchData, bool strict) override; private: - vespalib::ConstArrayRef<BitVector *> _bit_vectors; + AtomicBitVectorsRef _bit_vectors; bool _zeroHits; const BitVector* get_bit_vector(T value) const { static_assert(std::is_same_v<T, int8_t>, "Flag attribute search context is only supported for int8_t data type"); - return _bit_vectors[value + 128]; + return _bit_vectors[value + 128].load_acquire(); } template <class SC> friend class ::search::FlagAttributeIteratorT; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp index 8accb1123af..c17627a5026 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.cpp @@ -33,7 +33,7 @@ PostingListSearchContext(const IEnumStoreDictionary& dictionary, _FSTC(0.0), _PLSTC(0.0), _minBvDocFreq(minBvDocFreq), - _gbv(nullptr), + _bv(nullptr), _baseSearchCtx(baseSearchCtx) { } diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h index 9ee56a27a85..61bd8e2e2bb 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.h @@ -45,7 +45,7 @@ protected: float _FSTC; // Filtering Search Time Constant float _PLSTC; // Posting List Search Time Constant uint32_t _minBvDocFreq; - const GrowableBitVector *_gbv; // bitvector if _useBitVector has been set + const BitVector *_bv; // bitvector if _useBitVector has been set const ISearchContext &_baseSearchCtx; @@ -89,8 +89,6 @@ protected: return (numHits > 1000) && (calculateFilteringCost() < calculatePostingListCost(numHits)); } - -public: }; diff --git a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp index cd783ec2fe8..95bbaf27c26 100644 --- a/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postinglistsearchcontext.hpp @@ -45,21 +45,10 @@ PostingListSearchContextT<DataT>::lookupSingle() if (_postingList.isBitVector(typeId)) { const BitVectorEntry *bve = _postingList.getBitVectorEntry(_pidx); const GrowableBitVector *bv = bve->_bv.get(); - if (_useBitVector) { - _gbv = bv; - } else { - _pidx = bve->_tree; - if (_pidx.valid()) { - auto frozenView = _postingList.getTreeEntry(_pidx)->getFrozenView(_postingList.getAllocator()); - _frozenRoot = frozenView.getRoot(); - if (!_frozenRoot.valid()) { - _pidx = vespalib::datastore::EntryRef(); - } - } else { - _gbv = bv; - } - } - } else { + _bv = &bv->reader(); + _pidx = bve->_tree; + } + if (_pidx.valid()) { auto frozenView = _postingList.getTreeEntry(_pidx)->getFrozenView(_postingList.getAllocator()); _frozenRoot = frozenView.getRoot(); if (!_frozenRoot.valid()) { @@ -179,8 +168,8 @@ createPostingIterator(fef::TermFieldMatchData *matchData, bool strict) return search::BitVectorIterator::create(bv, bv->size(), *matchData, strict); } if (_uniqueValues == 1) { - if (_gbv != nullptr) { - return BitVectorIterator::create(_gbv, std::min(_gbv->size(), _docIdLimit), *matchData, strict); + if (_bv != nullptr && (!_pidx.valid() || _useBitVector || matchData->isNotNeeded())) { + return BitVectorIterator::create(_bv, std::min(_bv->size(), _docIdLimit), *matchData, strict); } if (!_pidx.valid()) { return std::make_unique<EmptySearch>(); @@ -217,9 +206,9 @@ template <typename DataT> unsigned int PostingListSearchContextT<DataT>::singleHits() const { - if (_gbv) { + if (_bv && !_pidx.valid()) { // Some inaccuracy is expected, data changes underfeet - return _gbv->countTrueBits(); + return _bv->countTrueBits(); } if (!_pidx.valid()) { return 0u; diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp index df016b050af..a942e70085a 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.cpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.cpp @@ -101,7 +101,7 @@ PostingStore<DataT>::removeSparseBitVectors() assert(isBitVector(typeId)); BitVectorEntry *bve = getWBitVectorEntry(iRef); GrowableBitVector &bv = *bve->_bv.get(); - uint32_t docFreq = bv.countTrueBits(); + uint32_t docFreq = bv.writer().countTrueBits(); if (bve->_tree.valid()) { RefType iRef2(bve->_tree); assert(isBTree(iRef2)); @@ -111,19 +111,19 @@ PostingStore<DataT>::removeSparseBitVectors() } if (docFreq < _minBvDocFreq) needscan = true; - unsigned int oldExtraSize = bv.extraByteSize(); - if (bv.size() > _bvSize) { + unsigned int oldExtraSize = bv.writer().extraByteSize(); + if (bv.writer().size() > _bvSize) { bv.shrink(_bvSize); res = true; } - if (bv.capacity() < _bvCapacity) { + if (bv.writer().capacity() < _bvCapacity) { bv.reserve(_bvCapacity); res = true; } - if (bv.size() < _bvSize) { + if (bv.writer().size() < _bvSize) { bv.extend(_bvSize); } - unsigned int newExtraSize = bv.extraByteSize(); + unsigned int newExtraSize = bv.writer().extraByteSize(); if (oldExtraSize != newExtraSize) { _bvExtraBytes = _bvExtraBytes + newExtraSize - oldExtraSize; } @@ -149,7 +149,7 @@ PostingStore<DataT>::consider_remove_sparse_bitvector(std::vector<EntryRef>& ref assert(isBitVector(typeId)); assert(_bvs.find(iRef.ref()) != _bvs.end()); BitVectorEntry *bve = getWBitVectorEntry(iRef); - BitVector &bv = *bve->_bv.get(); + BitVector &bv = bve->_bv->writer(); uint32_t docFreq = bv.countTrueBits(); if (bve->_tree.valid()) { RefType iRef2(bve->_tree); @@ -226,12 +226,12 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref) assert(isBitVector(typeId)); (void) typeId; BitVectorEntry *bve = getWBitVectorEntry(iRef); - AllocatedBitVector *bv = bve->_bv.get(); + GrowableBitVector *bv = bve->_bv.get(); assert(bv); - uint32_t docFreq = bv->countTrueBits(); + uint32_t docFreq = bv->writer().countTrueBits(); EntryRef ref2(bve->_tree); if (!ref2.valid()) { - makeDegradedTree(ref2, *bv); + makeDegradedTree(ref2, bv->writer()); } assert(ref2.valid()); assert(isBTree(ref2)); @@ -242,7 +242,7 @@ PostingStore<DataT>::dropBitVector(EntryRef &ref) _bvs.erase(ref.ref()); _store.holdElem(iRef, 1); _status.decBitVectors(); - _bvExtraBytes -= bv->extraByteSize(); + _bvExtraBytes -= bv->writer().extraByteSize(); ref = ref2; } @@ -258,7 +258,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) (void) typeId; vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); - AllocatedBitVector &bv = *bvsp.get(); + BitVector &bv = bvsp->writer(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; Iterator it = begin(ref); @@ -283,7 +283,7 @@ PostingStore<DataT>::makeBitVector(EntryRef &ref) bve->_bv = bvsp; _bvs.insert(bPair.ref.ref()); _status.incBitVectors(); - _bvExtraBytes += bv.extraByteSize(); + _bvExtraBytes += bvsp->writer().extraByteSize(); // barrier ? ref = bPair.ref; } @@ -299,7 +299,7 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, RefType iRef(ref); vespalib::GenerationHolder &genHolder = _store.getGenerationHolder(); auto bvsp = std::make_shared<GrowableBitVector>(_bvSize, _bvCapacity, genHolder); - AllocatedBitVector &bv = *bvsp.get(); + BitVector &bv = bvsp->writer(); uint32_t docIdLimit = _bvSize; (void) docIdLimit; uint32_t expDocFreq = ae - aOrg; @@ -319,7 +319,7 @@ PostingStore<DataT>::applyNewBitVector(EntryRef &ref, bve->_bv = bvsp; _bvs.insert(bPair.ref.ref()); _status.incBitVectors(); - _bvExtraBytes += bv.extraByteSize(); + _bvExtraBytes += bvsp->writer().extraByteSize(); // barrier ? ref = bPair.ref; } @@ -390,7 +390,7 @@ PostingStore<DataT>::apply(EntryRef &ref, BTreeType *tree = getWTreeEntry(iRef2); applyTree(tree, a, ae, r, re, CompareT()); } - BitVector *bv = bve->_bv.get(); + BitVector *bv = &bve->_bv->writer(); assert(bv); apply(*bv, a, ae, r, re); uint32_t docFreq = bv->countTrueBits(); @@ -433,7 +433,7 @@ PostingStore<DataT>::internalSize(uint32_t typeId, const RefType & iRef) const const BTreeType *tree = getTreeEntry(iRef2); return tree->size(_allocator); } else { - const BitVector *bv = bve->_bv.get(); + const BitVector *bv = &bve->_bv->writer(); return bv->countTrueBits(); } } else { @@ -456,7 +456,7 @@ PostingStore<DataT>::internalFrozenSize(uint32_t typeId, const RefType & iRef) c return tree->frozenSize(_allocator); } else { // Some inaccuracy is expected, data changes underfeet - return bve->_bv->countTrueBits(); + return bve->_bv->reader().countTrueBits(); } } else { const BTreeType *tree = getTreeEntry(iRef); @@ -608,7 +608,7 @@ PostingStore<DataT>::clear(const EntryRef ref) } _bvs.erase(ref.ref()); _status.decBitVectors(); - _bvExtraBytes -= bve->_bv->extraByteSize(); + _bvExtraBytes -= bve->_bv->writer().extraByteSize(); _store.holdElem(ref, 1); } else { BTreeType *tree = getWTreeEntry(iRef); diff --git a/searchlib/src/vespa/searchlib/attribute/postingstore.hpp b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp index 1993714e49d..7dab24e996f 100644 --- a/searchlib/src/vespa/searchlib/attribute/postingstore.hpp +++ b/searchlib/src/vespa/searchlib/attribute/postingstore.hpp @@ -25,7 +25,7 @@ PostingStore<DataT>::foreach_frozen_key(EntryRef ref, FunctionType func) const { const BTreeType *tree = getTreeEntry(iRef2); _allocator.getNodeStore().foreach_key(tree->getFrozenRoot(), func); } else { - const BitVector *bv = bve->_bv.get(); + const BitVector *bv = &bve->_bv->reader(); uint32_t docIdLimit = bv->size(); uint32_t docId = bv->getFirstTrueBit(1); while (docId < docIdLimit) { @@ -67,7 +67,7 @@ PostingStore<DataT>::foreach_frozen(EntryRef ref, FunctionType func) const { const BTreeType *tree = getTreeEntry(iRef2); _allocator.getNodeStore().foreach(tree->getFrozenRoot(), func); } else { - const BitVector *bv = bve->_bv.get(); + const BitVector *bv = &bve->_bv->reader(); uint32_t docIdLimit = bv->size(); uint32_t docId = bv->getFirstTrueBit(1); while (docId < docIdLimit) { diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp index 8db99c115a0..b0765680f78 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.cpp @@ -32,7 +32,7 @@ SingleBoolAttribute::~SingleBoolAttribute() void SingleBoolAttribute::ensureRoom(DocId docIdLimit) { - if (_bv.capacity() < docIdLimit) { + if (_bv.writer().capacity() < docIdLimit) { const GrowStrategy & gs = this->getConfig().getGrowStrategy(); uint32_t newSize = docIdLimit + (docIdLimit * gs.getDocsGrowFactor()) + gs.getDocsGrowDelta(); bool incGen = _bv.reserve(newSize); @@ -72,7 +72,7 @@ SingleBoolAttribute::onCommit() { setBit(change._doc, val != 0); } else if (change._type == ChangeBase::CLEARDOC) { std::atomic_thread_fence(std::memory_order_release); - _bv.clearBitAndMaintainCount(change._doc); + _bv.writer().clearBitAndMaintainCount(change._doc); } } } @@ -91,11 +91,11 @@ SingleBoolAttribute::onAddDocs(DocId docIdLimit) { void SingleBoolAttribute::onUpdateStat() { vespalib::MemoryUsage usage; - usage.setAllocatedBytes(_bv.extraByteSize()); - usage.setUsedBytes(_bv.sizeBytes()); + usage.setAllocatedBytes(_bv.writer().extraByteSize()); + usage.setUsedBytes(_bv.writer().sizeBytes()); usage.mergeGenerationHeldBytes(getGenerationHolder().getHeldBytes()); usage.merge(this->getChangeVectorMemoryUsage()); - this->updateStatistics(_bv.size(), _bv.size(), usage.allocatedBytes(), usage.usedBytes(), + this->updateStatistics(_bv.writer().size(), _bv.writer().size(), usage.allocatedBytes(), usage.usedBytes(), usage.deadBytes(), usage.allocatedBytesOnHold()); } @@ -104,6 +104,7 @@ namespace { class BitVectorSearchContext : public attribute::SearchContext, public attribute::IPostingListSearchContext { private: + uint32_t _doc_id_limit; const BitVector & _bv; bool _invert; bool _valid; @@ -122,7 +123,7 @@ private: } public: - BitVectorSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const SingleBoolAttribute & bv); + BitVectorSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const SingleBoolAttribute & attr); std::unique_ptr<queryeval::SearchIterator> createFilterIterator(fef::TermFieldMatchData * matchData, bool strict) override; @@ -133,6 +134,7 @@ public: BitVectorSearchContext::BitVectorSearchContext(std::unique_ptr<QueryTermSimple> qTerm, const SingleBoolAttribute & attr) : SearchContext(attr), + _doc_id_limit(attr.getCommittedDocIdLimit()), _bv(attr.getBitVector()), _invert(false), _valid(qTerm->isValid()) @@ -152,7 +154,7 @@ BitVectorSearchContext::createFilterIterator(fef::TermFieldMatchData * matchData if (!valid()) { return std::make_unique<queryeval::EmptySearch>(); } - return BitVectorIterator::create(&_bv, _attr.getCommittedDocIdLimit(), *matchData, strict, _invert); + return BitVectorIterator::create(&_bv, _doc_id_limit, *matchData, strict, _invert); } void @@ -188,13 +190,13 @@ SingleBoolAttribute::onLoad(vespalib::Executor *) if (ok) { setCreateSerialNum(attrReader.getCreateSerialNum()); getGenerationHolder().clearHoldLists(); - _bv.clear(); + _bv.writer().clear(); uint32_t numDocs = attrReader.getNextData(); _bv.extend(numDocs); - ssize_t bytesRead = attrReader.getReader().read(_bv.getStart(), _bv.sizeBytes()); - _bv.invalidateCachedCount(); - _bv.countTrueBits(); - assert(bytesRead == _bv.sizeBytes()); + ssize_t bytesRead = attrReader.getReader().read(_bv.writer().getStart(), _bv.writer().sizeBytes()); + _bv.writer().invalidateCachedCount(); + _bv.writer().countTrueBits(); + assert(bytesRead == _bv.writer().sizeBytes()); setNumDocs(numDocs); setCommittedDocIdLimit(numDocs); } @@ -207,7 +209,7 @@ SingleBoolAttribute::onSave(IAttributeSaveTarget &saveTarget) { assert(!saveTarget.getEnumerated()); const size_t numDocs(getCommittedDocIdLimit()); - const size_t sz(sizeof(uint32_t) + _bv.sizeBytes()); + const size_t sz(sizeof(uint32_t) + _bv.writer().sizeBytes()); IAttributeSaveTarget::Buffer buf(saveTarget.datWriter().allocBuf(sz)); char *p = buf->getFree(); @@ -215,8 +217,8 @@ SingleBoolAttribute::onSave(IAttributeSaveTarget &saveTarget) uint32_t numDocs2 = numDocs; memcpy(p, &numDocs2, sizeof(uint32_t)); p += sizeof(uint32_t); - memcpy(p, _bv.getStart(), _bv.sizeBytes()); - p += _bv.sizeBytes(); + memcpy(p, _bv.writer().getStart(), _bv.writer().sizeBytes()); + p += _bv.writer().sizeBytes(); assert(p == e); (void) e; buf->moveFreeToData(sz); @@ -249,7 +251,7 @@ uint64_t SingleBoolAttribute::getEstimatedSaveByteSize() const { constexpr uint64_t headerSize = FileSettings::DIRECTIO_ALIGNMENT + sizeof(uint32_t); - return headerSize + _bv.sizeBytes(); + return headerSize + _bv.reader().sizeBytes(); } void diff --git a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h index 77c8b7e318f..3465269de8a 100644 --- a/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h +++ b/searchlib/src/vespa/searchlib/attribute/singleboolattribute.h @@ -34,7 +34,7 @@ public: getSearch(std::unique_ptr<QueryTermSimple> term, const attribute::SearchContextParams & params) const override; uint32_t getValueCount(DocId doc) const override { - return (doc >= _bv.size()) ? 0 : 1; + return (doc >= _bv.reader().size()) ? 0 : 1; } largeint_t getInt(DocId doc) const override { return static_cast<largeint_t>(getFast(doc)); @@ -81,12 +81,12 @@ public: int8_t get(DocId doc) const override { return getFast(doc); } - const BitVector & getBitVector() const { return _bv; } + const BitVector & getBitVector() const { return _bv.reader(); } void setBit(DocId doc, bool value) { if (value) { - _bv.setBitAndMaintainCount(doc); + _bv.writer().setBitAndMaintainCount(doc); } else { - _bv.clearBitAndMaintainCount(doc); + _bv.writer().clearBitAndMaintainCount(doc); } } protected: @@ -99,7 +99,7 @@ private: return 0; } int8_t getFast(DocId doc) const { - return _bv.testBit(doc) ? 1 : 0; + return _bv.reader().testBit(doc) ? 1 : 0; } vespalib::alloc::Alloc _init_alloc; GrowableBitVector _bv; diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp index f37968c70a0..33f044845aa 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.cpp @@ -108,23 +108,4 @@ AllocatedBitVector::resize(Index newLength) clear(); } -AllocatedBitVector & -AllocatedBitVector::operator=(const AllocatedBitVector & rhs) -{ - AllocatedBitVector tmp(rhs); - swap(tmp); - assert(testBit(size())); - - return *this; -} -AllocatedBitVector & -AllocatedBitVector::operator=(const BitVector & rhs) -{ - AllocatedBitVector tmp(rhs); - swap(tmp); - assert(testBit(size())); - - return *this; -} - } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h index 5106717dad0..a47082dc413 100644 --- a/searchlib/src/vespa/searchlib/common/allocatedbitvector.h +++ b/searchlib/src/vespa/searchlib/common/allocatedbitvector.h @@ -41,8 +41,6 @@ public: AllocatedBitVector(const BitVector &other); AllocatedBitVector(const AllocatedBitVector &other); ~AllocatedBitVector() override; - AllocatedBitVector &operator=(const AllocatedBitVector &other); - AllocatedBitVector &operator=(const BitVector &other); /** * Query the size of the bit vector. @@ -67,14 +65,8 @@ protected: private: friend class BitVectorTest; friend class GrowableBitVector; - void swap(AllocatedBitVector & rhs) { - std::swap(_capacityBits, rhs._capacityBits); - _alloc.swap(rhs._alloc); - BitVector::swap(rhs); - } AllocatedBitVector(const BitVector &other, std::pair<Index, Index> size_capacity); }; } // namespace search - diff --git a/searchlib/src/vespa/searchlib/common/bitvector.cpp b/searchlib/src/vespa/searchlib/common/bitvector.cpp index 36ebc6597fa..03fe97cabbb 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/bitvector.cpp @@ -41,9 +41,6 @@ constexpr size_t MMAP_LIMIT = 256_Mi; namespace search { using vespalib::nbostream; -using vespalib::GenerationHeldBase; -using vespalib::GenerationHeldAlloc; -using vespalib::GenerationHolder; Alloc BitVector::allocatePaddedAndAligned(Index start, Index end, Index capacity, const Alloc* init_alloc) @@ -105,13 +102,13 @@ BitVector::clearIntervalNoInvalidation(Range range_in) Index endw = wordNum(last); if (endw > startw) { - store_word(startw, _words[startw] & startBits(range.start())); + store(_words[startw], _words[startw] & startBits(range.start())); for (Index i = startw + 1; i < endw; ++i) { - store_word(i, 0); + store(_words[i], 0); } - store_word(endw, _words[endw] & endBits(last)); + store(_words[endw], _words[endw] & endBits(last)); } else { - store_word(startw, _words[startw] & (startBits(range.start()) | endBits(last))); + store(_words[startw], _words[startw] & (startBits(range.start()) | endBits(last))); } } @@ -126,11 +123,13 @@ BitVector::setInterval(Index start_in, Index end_in) Index endw = wordNum(last); if (endw > startw) { - _words[startw++] |= checkTab(range.start()); - memset(_words + startw, 0xff, sizeof(*_words)*(endw-startw)); - _words[endw] |= ~endBits(last); + store(_words[startw], _words[startw] | checkTab(range.start())); + for (Index i = startw + 1; i < endw; ++i) { + store(_words[i], allBits()); + } + store(_words[endw], _words[endw] | ~endBits(last)); } else { - _words[startw] |= ~(startBits(range.start()) | endBits(last)); + store(_words[startw], _words[startw] | ~(startBits(range.start()) | endBits(last))); } invalidateCachedCount(); @@ -155,17 +154,17 @@ BitVector::countInterval(Range range_in) const Word *bitValues = _words; if (startw == endw) { - return Optimized::popCount(bitValues[startw] & ~(startBits(range.start()) | endBits(last))); + return Optimized::popCount(load(bitValues[startw]) & ~(startBits(range.start()) | endBits(last))); } Index res = 0; // Limit to full words if ((range.start() & (WordLen - 1)) != 0) { - res += Optimized::popCount(bitValues[startw] & ~startBits(range.start())); + res += Optimized::popCount(load(bitValues[startw]) & ~startBits(range.start())); ++startw; } // Align start to 16 bytes while (startw < endw && (startw & 3) != 0) { - res += Optimized::popCount(bitValues[startw]); + res += Optimized::popCount(load(bitValues[startw])); ++startw; } bool partialEnd = (last & (WordLen - 1)) != (WordLen - 1); @@ -176,7 +175,7 @@ BitVector::countInterval(Range range_in) const res += IAccelrated::getAccelerator().populationCount(bitValues + startw, endw - startw); } if (partialEnd) { - res += Optimized::popCount(bitValues[endw] & ~endBits(last)); + res += Optimized::popCount(load(bitValues[endw]) & ~endBits(last)); } return res; @@ -194,7 +193,7 @@ BitVector::orWith(const BitVector & right) IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); } Index last(right.size() - 1); - getWordIndex(last)[0] |= (right.getWordIndex(last)[0] & ~endBits(last)); + store(getWordIndex(last)[0], getWordIndex(last)[0] | (load(right.getWordIndex(last)[0]) & ~endBits(last))); } } else { IAccelrated::getAccelerator().orBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); @@ -209,8 +208,8 @@ BitVector::repairEnds() if (size() != 0) { Index start(getStartIndex()); Index last(size() - 1); - getWordIndex(start)[0] &= ~startBits(start); - getWordIndex(last)[0] &= ~endBits(last); + store(getWordIndex(start)[0], getWordIndex(start)[0] & ~startBits(start)); + store(getWordIndex(last)[0], getWordIndex(last)[0] & ~endBits(last)); } setGuardBit(); } @@ -244,7 +243,7 @@ BitVector::andNotWith(const BitVector& right) IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), commonBytes); } Index last(right.size() - 1); - getWordIndex(last)[0] &= ~(right.getWordIndex(last)[0] & ~endBits(last)); + store(getWordIndex(last)[0], getWordIndex(last)[0] & ~(load(right.getWordIndex(last)[0]) & ~endBits(last))); } } else { IAccelrated::getAccelerator().andNotBit(getActiveStart(), right.getWordIndex(getStartIndex()), getActiveBytes()); @@ -272,7 +271,7 @@ BitVector::operator==(const BitVector &rhs) const const Word *words = getActiveStart(); const Word *oWords = rhs.getActiveStart(); for (Index i = 0; i < bitVectorSize; i++) { - if (words[i] != oWords[i]) { + if (load(words[i]) != load(oWords[i])) { return false; } } @@ -285,13 +284,13 @@ BitVector::hasTrueBitsInternal() const Index bitVectorSizeL1(numActiveWords() - 1); const Word *words(getActiveStart()); for (Index i = 0; i < bitVectorSizeL1; i++) { - if (words[i] != 0) { + if (load(words[i]) != 0) { return true; } } // Ignore guard bit. - if ((words[bitVectorSizeL1] & ~mask(size())) != 0) + if ((load(words[bitVectorSizeL1]) & ~mask(size())) != 0) return true; return false; diff --git a/searchlib/src/vespa/searchlib/common/bitvector.h b/searchlib/src/vespa/searchlib/common/bitvector.h index 7fea95661d7..912c4a47c39 100644 --- a/searchlib/src/vespa/searchlib/common/bitvector.h +++ b/searchlib/src/vespa/searchlib/common/bitvector.h @@ -6,7 +6,6 @@ #include <memory> #include <vespa/vespalib/util/alloc.h> #include <vespa/vespalib/util/atomic.h> -#include <vespa/vespalib/util/generationholder.h> #include <vespa/fastos/types.h> namespace vespalib { @@ -24,8 +23,6 @@ class BitVector : protected BitWord { public: using Index = BitWord::Index; - using GenerationHolder = vespalib::GenerationHolder; - using GenerationHeldBase = vespalib::GenerationHeldBase; using UP = std::unique_ptr<BitVector>; class Range { public: @@ -45,17 +42,14 @@ public: void * getStart() { return _words; } Index size() const { return vespalib::atomic::load_ref_relaxed(_sz); } Index sizeBytes() const { return numBytes(getActiveSize()); } - Word load_word(Index widx) const { return vespalib::atomic::load_ref_relaxed(_words[widx]); } - void store_word(Index widx, Word word) { return vespalib::atomic::store_ref_relaxed(_words[widx], word); } bool testBit(Index idx) const { - return ((load_word(wordNum(idx)) & mask(idx)) != 0); + return ((load(_words[wordNum(idx)]) & mask(idx)) != 0); } - Index getSizeSafe() const { + Index getSizeAcquire() const { return vespalib::atomic::load_ref_acquire(_sz); } - bool testBitSafe(Index idx) const { - auto my_words = vespalib::atomic::load_ref_acquire(_words); - auto my_word = vespalib::atomic::load_ref_acquire(my_words[wordNum(idx)]); + bool testBitAcquire(Index idx) const { + auto my_word = vespalib::atomic::load_ref_acquire(_words[wordNum(idx)]); return (my_word & mask(idx)) != 0; } bool hasTrueBits() const { @@ -127,10 +121,10 @@ public: Index getPrevTrueBit(Index start) const { Index index(wordNum(start)); const Word *words(_words); - Word t(words[index] & ~endBits(start)); + Word t(load(words[index]) & ~endBits(start)); while(t == 0 && index > getStartWordNum()) { - t = words[--index]; + t = load(words[--index]); } return (t != 0) @@ -148,13 +142,13 @@ public: vespalib::atomic::store_ref_release(_sz, sz); } void setBit(Index idx) { - store_word(wordNum(idx), _words[wordNum(idx)] | mask(idx)); + store(_words[wordNum(idx)], _words[wordNum(idx)] | mask(idx)); } void clearBit(Index idx) { - store_word(wordNum(idx), _words[wordNum(idx)] & ~ mask(idx)); + store(_words[wordNum(idx)], _words[wordNum(idx)] & ~ mask(idx)); } void flipBit(Index idx) { - _words[wordNum(idx)] ^= mask(idx); + store(_words[wordNum(idx)], _words[wordNum(idx)] ^ mask(idx)); } void andWith(const BitVector &right); @@ -211,17 +205,6 @@ public: _numTrueBits.store(invalidCount(), std::memory_order_relaxed); } - void swap(BitVector & rhs) { - auto my_words = _words; - vespalib::atomic::store_ref_release(_words, rhs._words); - vespalib::atomic::store_ref_release(rhs._words, my_words); - std::swap(_startOffset, rhs._startOffset); - std::swap(_sz, rhs._sz); - Index tmp = rhs._numTrueBits; - rhs._numTrueBits = _numTrueBits.load(std::memory_order_relaxed); - _numTrueBits.store(tmp, std::memory_order_relaxed); - } - /** * Count bits in partial bitvector [..>. * @@ -292,6 +275,8 @@ protected: static Alloc allocatePaddedAndAligned(Index start, Index end, Index capacity, const Alloc* init_alloc = nullptr); private: + Word load(const Word &word) const { return vespalib::atomic::load_ref_relaxed(word); } + void store(Word &word, Word value) { return vespalib::atomic::store_ref_relaxed(word, value); } friend PartialBitVector; const Word * getWordIndex(Index index) const { return static_cast<const Word *>(getStart()) + wordNum(index); } Word * getWordIndex(Index index) { return static_cast<Word *>(getStart()) + wordNum(index); } @@ -334,8 +319,8 @@ private: Index index(wordNum(start)); Index lastIndex(wordNum(last)); - Word word(conv(_words[index]) & checkTab(start)); - for ( ; index < lastIndex; word = conv(_words[++index])) { + Word word(conv(load(_words[index])) & checkTab(start)); + for ( ; index < lastIndex; word = conv(load(_words[++index]))) { foreach_bit(func, word, index << numWordBits()); } foreach_bit(func, word & ~endBits(last), lastIndex << numWordBits()); @@ -344,14 +329,14 @@ private: Index getNextBit(WordConverter conv, Index start) const { Index index(wordNum(start)); const Word *words(_words); - Word t(conv(words[index]) & checkTab(start)); + Word t(conv(load(words[index])) & checkTab(start)); // In order to avoid a test an extra guard bit is added // after the bitvector as a termination. // Also bitvector will normally at least 1 bit set per 32 bits. // So that is what we should expect. while (__builtin_expect(t == 0, false)) { - t = conv(words[++index]); + t = conv(load(words[++index])); } return (index << numWordBits()) + vespalib::Optimized::lsbIdx(t); @@ -397,4 +382,3 @@ void BitVector::andNotWithT(T it) { } } // namespace search - diff --git a/searchlib/src/vespa/searchlib/common/bitword.h b/searchlib/src/vespa/searchlib/common/bitword.h index 96665d47c93..d90ec9c93d3 100644 --- a/searchlib/src/vespa/searchlib/common/bitword.h +++ b/searchlib/src/vespa/searchlib/common/bitword.h @@ -17,6 +17,7 @@ public: static constexpr size_t WordLen = sizeof(Word)*8; static uint8_t bitNum(Index idx) { return (idx % WordLen); } static Word endBits(Index index) { return (std::numeric_limits<Word>::max() - 1) << bitNum(index); } + static Word allBits() { return std::numeric_limits<Word>::max(); } static Index wordNum(Index idx) { return idx >> numWordBits(); } static Word mask(Index idx) { return Word(1) << bitNum(idx); } static constexpr uint8_t size_bits(uint8_t n) { return (n > 1) ? (1 + size_bits(n >> 1)) : 0; } diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp index 7a548b3e7ac..e3334be3fd9 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.cpp +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.cpp @@ -7,52 +7,65 @@ namespace search { using vespalib::GenerationHeldBase; -using vespalib::GenerationHeldAlloc; using vespalib::GenerationHolder; +namespace { + +struct GenerationHeldAllocatedBitVector : public vespalib::GenerationHeldBase { + std::unique_ptr<AllocatedBitVector> vector; + GenerationHeldAllocatedBitVector(std::unique_ptr<AllocatedBitVector> vector_in) + : GenerationHeldBase(sizeof(AllocatedBitVector) + vector_in->extraByteSize()), + vector(std::move(vector_in)) {} +}; + +} + GenerationHeldBase::UP -GrowableBitVector::grow(Index newSize, Index newCapacity) +GrowableBitVector::grow(BitWord::Index newSize, BitWord::Index newCapacity) { + AllocatedBitVector &self = *_stored; assert(newCapacity >= newSize); - GenerationHeldBase::UP ret; - if (newCapacity != capacity()) { - AllocatedBitVector tbv(newSize, newCapacity, _alloc.get(), size(), &_alloc); - if (newSize > size()) { - tbv.clearBitAndMaintainCount(size()); // Clear old guard bit. + if (newCapacity != self.capacity()) { + auto tbv = std::make_unique<AllocatedBitVector>(newSize, newCapacity, self._alloc.get(), self.size(), &self._alloc); + if (newSize > self.size()) { + tbv->clearBitAndMaintainCount(self.size()); // Clear old guard bit. } - ret = std::make_unique<GenerationHeldAlloc<Alloc>>(_alloc); - swap(tbv); + auto to_hold = std::make_unique<GenerationHeldAllocatedBitVector>(std::move(_stored)); + _self.store(tbv.get(), std::memory_order_release); + _stored = std::move(tbv); + return to_hold; } else { - if (newSize > size()) { - Range clearRange(size(), newSize); - setSize(newSize); - clearIntervalNoInvalidation(clearRange); + if (newSize > self.size()) { + BitVector::Range clearRange(self.size(), newSize); + self.setSize(newSize); + self.clearIntervalNoInvalidation(clearRange); } else { - clearIntervalNoInvalidation(Range(newSize, size())); - setSize(newSize); - updateCount(); + self.clearIntervalNoInvalidation(BitVector::Range(newSize, self.size())); + self.setSize(newSize); + self.updateCount(); } } - return ret; + return {}; } -GrowableBitVector::GrowableBitVector(Index newSize, Index newCapacity, +GrowableBitVector::GrowableBitVector(BitWord::Index newSize, BitWord::Index newCapacity, GenerationHolder &generationHolder, - const Alloc* init_alloc) - : AllocatedBitVector(newSize, newCapacity, nullptr, 0, init_alloc), - _generationHolder(generationHolder) + const Alloc *init_alloc) + : _stored(std::make_unique<AllocatedBitVector>(newSize, newCapacity, nullptr, 0, init_alloc)), + _self(_stored.get()), + _generationHolder(generationHolder) { assert(newSize <= newCapacity); } bool -GrowableBitVector::reserve(Index newCapacity) +GrowableBitVector::reserve(BitWord::Index newCapacity) { - Index oldCapacity = capacity(); + BitWord::Index oldCapacity = _stored->capacity(); assert(newCapacity >= oldCapacity); if (newCapacity == oldCapacity) return false; - return hold(grow(size(), newCapacity)); + return hold(grow(_stored->size(), newCapacity)); } bool @@ -66,18 +79,18 @@ GrowableBitVector::hold(GenerationHeldBase::UP v) } bool -GrowableBitVector::shrink(Index newCapacity) +GrowableBitVector::shrink(BitWord::Index newCapacity) { - Index oldCapacity = capacity(); + BitWord::Index oldCapacity = _stored->capacity(); assert(newCapacity <= oldCapacity); (void) oldCapacity; - return hold(grow(newCapacity, std::max(capacity(), newCapacity))); + return hold(grow(newCapacity, std::max(_stored->capacity(), newCapacity))); } bool -GrowableBitVector::extend(Index newCapacity) +GrowableBitVector::extend(BitWord::Index newCapacity) { - return hold(grow(newCapacity, std::max(capacity(), newCapacity))); + return hold(grow(newCapacity, std::max(_stored->capacity(), newCapacity))); } } // namespace search diff --git a/searchlib/src/vespa/searchlib/common/growablebitvector.h b/searchlib/src/vespa/searchlib/common/growablebitvector.h index 8773f1573d6..e9443512040 100644 --- a/searchlib/src/vespa/searchlib/common/growablebitvector.h +++ b/searchlib/src/vespa/searchlib/common/growablebitvector.h @@ -3,22 +3,39 @@ #pragma once #include "allocatedbitvector.h" +#include <vespa/vespalib/util/atomic.h> +#include <vespa/vespalib/util/generationholder.h> namespace search { -class GrowableBitVector : public AllocatedBitVector +class GrowableBitVector { public: - GrowableBitVector(Index newSize, Index newCapacity, GenerationHolder &generationHolder, const Alloc* init_alloc = nullptr); + using Alloc = vespalib::alloc::Alloc; + using GenerationHolder = vespalib::GenerationHolder; + using GenerationHeldBase = vespalib::GenerationHeldBase; + GrowableBitVector(BitWord::Index newSize, BitWord::Index newCapacity, + GenerationHolder &generationHolder, const Alloc *init_alloc = nullptr); + + const BitVector &reader() const { return acquire_self(); } + AllocatedBitVector &writer() { return *_stored; } + + BitWord::Index extraByteSize() const { + return sizeof(AllocatedBitVector) + acquire_self().extraByteSize(); + } /** Will return true if a a buffer is held */ - bool reserve(Index newCapacity); - bool shrink(Index newCapacity); - bool extend(Index newCapacity); + bool reserve(BitWord::Index newCapacity); + bool shrink(BitWord::Index newCapacity); + bool extend(BitWord::Index newCapacity); private: - GenerationHeldBase::UP grow(Index newLength, Index newCapacity); + GenerationHeldBase::UP grow(BitWord::Index newLength, BitWord::Index newCapacity); + + AllocatedBitVector &acquire_self() const { return *(_self.load(std::memory_order_acquire)); } VESPA_DLL_LOCAL bool hold(GenerationHeldBase::UP v); + std::unique_ptr<AllocatedBitVector> _stored; + std::atomic<AllocatedBitVector *> _self; GenerationHolder &_generationHolder; }; diff --git a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h index d663d1659e1..5c5791f585a 100644 --- a/searchlib/src/vespa/searchlib/common/idocumentmetastore.h +++ b/searchlib/src/vespa/searchlib/common/idocumentmetastore.h @@ -5,7 +5,8 @@ #include "lid_usage_stats.h" #include <vespa/document/base/globalid.h> #include <vespa/document/bucket/bucketid.h> -#include <persistence/spi/types.h> +#include <vector> +#include <memory> namespace search { @@ -15,7 +16,7 @@ namespace search { struct DocumentMetaData { typedef uint32_t DocId; DocId lid; - storage::spi::Timestamp timestamp; + uint64_t timestamp; document::BucketId bucketId; document::GlobalId gid; bool removed; @@ -31,14 +32,14 @@ struct DocumentMetaData { { } DocumentMetaData(DocId lid_, - storage::spi::Timestamp timestamp_, + uint64_t timestamp_, document::BucketId bucketId_, const document::GlobalId &gid_) noexcept : DocumentMetaData(lid_, timestamp_, bucketId_, gid_, false) { } DocumentMetaData(DocId lid_, - storage::spi::Timestamp timestamp_, + uint64_t timestamp_, document::BucketId bucketId_, const document::GlobalId &gid_, bool removed_) noexcept @@ -72,7 +73,7 @@ struct IDocumentMetaStore { typedef uint32_t DocId; typedef document::GlobalId GlobalId; typedef document::BucketId BucketId; - typedef storage::spi::Timestamp Timestamp; + typedef uint64_t Timestamp; virtual ~IDocumentMetaStore() {} diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h index 7f44728e1be..b1e5ca55a15 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectordictionary.h @@ -6,6 +6,7 @@ #include <vespa/searchlib/index/bitvectorkeys.h> #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> +#include <vector> namespace search::diskindex { diff --git a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h index 007368babdd..162ca512cec 100644 --- a/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h +++ b/searchlib/src/vespa/searchlib/diskindex/bitvectorfile.h @@ -7,6 +7,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/stllike/allocator.h> +#include <vector> class Fast_BufferedFile; diff --git a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp index b991773c50f..51a4d392839 100644 --- a/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp +++ b/searchlib/src/vespa/searchlib/docstore/logdatastore.cpp @@ -450,7 +450,10 @@ void LogDataStore::compactFile(FileId fileId) IWriteData::UP compacter; FileId destinationFileId = FileId::active(); if (_bucketizer) { - if ( ! shouldCompactToActiveFile(fc->getDiskFootprint() - fc->getDiskBloat())) { + size_t disk_footprint = fc->getDiskFootprint(); + size_t disk_bloat = fc->getDiskBloat(); + size_t compacted_size = (disk_footprint <= disk_bloat) ? 0u : (disk_footprint - disk_bloat); + if ( ! shouldCompactToActiveFile(compacted_size)) { MonitorGuard guard(_updateLock); destinationFileId = allocateFileId(guard); setNewFileChunk(guard, createWritableFile(destinationFileId, fc->getLastPersistedSerialNum(), fc->getNameId().next())); @@ -464,9 +467,8 @@ void LogDataStore::compactFile(FileId fileId) fc->appendTo(_executor, *this, *compacter, fc->getNumChunks(), nullptr, CpuCategory::COMPACT); - if (destinationFileId.isActive()) { - flushActiveAndWait(0); - } else { + flushActiveAndWait(0); + if (!destinationFileId.isActive()) { MonitorGuard guard(_updateLock); auto & compactTo = dynamic_cast<WriteableFileChunk &>(*_fileChunks[destinationFileId.getId()]); flushFileAndWait(std::move(guard), compactTo, 0); diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.cpp b/searchlib/src/vespa/searchlib/features/queryfeature.cpp index cbf7956e09b..483ba6f82b4 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/queryfeature.cpp @@ -14,6 +14,8 @@ #include <vespa/eval/eval/value_type.h> #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/fast_value.h> +#include <vespa/eval/eval/function.h> +#include <vespa/eval/eval/interpreted_function.h> #include <vespa/vespalib/locale/c.h> #include <vespa/vespalib/util/issue.h> #include <cerrno> @@ -26,6 +28,11 @@ using namespace search::fef::indexproperties; using document::TensorDataType; using vespalib::eval::ValueType; using vespalib::eval::Value; +using vespalib::eval::TensorSpec; +using vespalib::eval::Function; +using vespalib::eval::InterpretedFunction; +using vespalib::eval::NodeTypes; +using vespalib::eval::SimpleObjectParams; using vespalib::Issue; using search::fef::FeatureType; using search::fef::AnyWrapper; @@ -61,6 +68,29 @@ feature_t asFeature(const vespalib::string &str) { return val; } +// Create an empty tensor of the given type. +Value::UP empty_tensor(const ValueType &type) { + const auto &factory = vespalib::eval::FastValueBuilderFactory::get(); + return vespalib::eval::value_from_spec(TensorSpec(type.to_spec()), factory); +} + +// Create a tensor value by evaluating a self-contained expression. +Value::UP as_tensor(const vespalib::string &expr, const ValueType &wanted_type) { + const auto &factory = vespalib::eval::FastValueBuilderFactory::get(); + auto fun = Function::parse(expr); + if (!fun->has_error() && (fun->num_params() == 0)) { + NodeTypes types = NodeTypes(*fun, {}); + ValueType res_type = types.get_type(fun->root()); + if (res_type == wanted_type) { + SimpleObjectParams params({}); + InterpretedFunction ifun(factory, *fun, types); + InterpretedFunction::Context ctx(ifun); + return factory.copy(ifun.eval(ctx, params)); + } + } + return {}; +} + // query(foo): // query.value.foo -> decoded tensor value 'foo' vespalib::string make_value_key(const vespalib::string &base, const vespalib::string &sub_key) { @@ -72,12 +102,39 @@ vespalib::string make_value_key(const vespalib::string &base, const vespalib::st } // namespace search::features::<unnamed> -QueryBlueprint::QueryBlueprint() : - Blueprint("query"), +Property +QueryBlueprint::config_lookup(const IIndexEnvironment &env) const +{ + const auto &props = env.getProperties(); + auto res = props.lookup(getName()); // query(foo) + if (!res.found()) { + res = props.lookup(_old_key); // $foo + } + return res; +} + +Property +QueryBlueprint::request_lookup(const IQueryEnvironment &env) const +{ + const auto &props = env.getProperties(); + auto res = props.lookup(getName()); // query(foo) + if (!res.found()) { + res = props.lookup(_key); // foo + } + if (!res.found()) { + res = props.lookup(_old_key); // $foo + } + return res; +} + +QueryBlueprint::QueryBlueprint() + : Blueprint("query"), _key(), - _key2(), - _defaultValue(0), - _valueType(ValueType::double_type()) + _old_key(), + _stored_value_key(), + _type(ValueType::double_type()), + _default_number_value(), + _default_object_value() { } @@ -98,42 +155,41 @@ bool QueryBlueprint::setup(const IIndexEnvironment &env, const ParameterList ¶ms) { _key = params[0].getValue(); - _key2 = "$"; - _key2.append(_key); + _old_key = "$"; + _old_key.append(_key); _stored_value_key = make_value_key(getBaseName(), _key); - - vespalib::string key3; - key3.append("query("); - key3.append(_key); - key3.append(")"); - Property p = env.getProperties().lookup(key3); - if (!p.found()) { - p = env.getProperties().lookup(_key2); - } - if (p.found()) { - _defaultValue = asFeature(p.get()); + vespalib::string type_str = type::QueryFeature::lookup(env.getProperties(), _key); + if (!type_str.empty()) { + _type = ValueType::from_spec(type_str); + if (_type.is_error()) { + return fail("invalid type: '%s'", type_str.c_str()); + } } - vespalib::string queryFeatureType = type::QueryFeature::lookup(env.getProperties(), _key); - if (!queryFeatureType.empty()) { - _valueType = ValueType::from_spec(queryFeatureType); - if (_valueType.is_error()) { - LOG(error, "%s: invalid type: '%s'", getName().c_str(), queryFeatureType.c_str()); + Property p = config_lookup(env); + if (_type.is_double()) { + if (p.found()) { + _default_number_value = asFeature(p.get()); + } + } else { + if (p.found()) { + _default_object_value = as_tensor(p.get(), _type); + if (_default_object_value.get() == nullptr) { + return fail("could not create default tensor value of type '%s' from the expression '%s'", + _type.to_spec().c_str(), p.get().c_str()); + } + } else { + _default_object_value = empty_tensor(_type); } } - FeatureType output_type = _valueType.is_double() - ? FeatureType::number() - : FeatureType::object(_valueType); + FeatureType output_type = _type.is_double() ? FeatureType::number() : FeatureType::object(_type); describeOutput("out", "The value looked up in query properties using the given key.", output_type); - return !_valueType.is_error(); + assert(_type.has_dimensions() == (_default_object_value.get() != nullptr)); + return true; } namespace { -Value::UP make_tensor_value(const IQueryEnvironment &env, - const vespalib::string &queryKey, - const ValueType &valueType) -{ - Property prop = env.getProperties().lookup(queryKey); +Value::UP decode_tensor_value(Property prop, const ValueType &valueType) { if (prop.found() && !prop.get().empty()) { const vespalib::string &value = prop.get(); vespalib::nbostream stream(value.data(), value.size()); @@ -157,9 +213,8 @@ Value::UP make_tensor_value(const IQueryEnvironment &env, void QueryBlueprint::prepareSharedState(const fef::IQueryEnvironment &env, fef::IObjectStore &store) const { - if (!_stored_value_key.empty() && _valueType.has_dimensions() && (store.get(_stored_value_key) == nullptr)) { - auto value = make_tensor_value(env, _key, _valueType); - if (value) { + if (!_stored_value_key.empty() && _type.has_dimensions() && (store.get(_stored_value_key) == nullptr)) { + if (auto value = decode_tensor_value(request_lookup(env), _type)) { store.add(_stored_value_key, std::make_unique<ValueWrapper>(std::move(value))); } } @@ -168,23 +223,19 @@ QueryBlueprint::prepareSharedState(const fef::IQueryEnvironment &env, fef::IObje FeatureExecutor & QueryBlueprint::createExecutor(const IQueryEnvironment &env, vespalib::Stash &stash) const { - if (_valueType.has_dimensions()) { + if (_type.has_dimensions()) { if (const Anything *wrapped_value = env.getObjectStore().get(_stored_value_key)) { if (const Value *value = ValueWrapper::getValue(*wrapped_value).get()) { return stash.create<ConstantTensorRefExecutor>(*value); } } - return ConstantTensorExecutor::createEmpty(_valueType, stash); + return stash.create<ConstantTensorRefExecutor>(*_default_object_value); } else { - std::vector<feature_t> values; - Property p = env.getProperties().lookup(_key); - if (!p.found()) { - p = env.getProperties().lookup(_key2); - } + auto p = request_lookup(env); if (p.found()) { return stash.create<SingleValueExecutor>(asFeature(p.get())); } else { - return stash.create<SingleValueExecutor>(_defaultValue); + return stash.create<SingleValueExecutor>(_default_number_value); } } } diff --git a/searchlib/src/vespa/searchlib/features/queryfeature.h b/searchlib/src/vespa/searchlib/features/queryfeature.h index 05e44bcd923..020fd31989d 100644 --- a/searchlib/src/vespa/searchlib/features/queryfeature.h +++ b/searchlib/src/vespa/searchlib/features/queryfeature.h @@ -3,7 +3,9 @@ #pragma once #include <vespa/searchlib/fef/blueprint.h> +#include <vespa/searchlib/fef/properties.h> #include <vespa/eval/eval/value_type.h> +#include <vespa/eval/eval/value.h> namespace search::features { @@ -15,11 +17,15 @@ namespace search::features { */ class QueryBlueprint : public fef::Blueprint { private: - vespalib::string _key; // 'foo' - vespalib::string _key2; // '$foo' - vespalib::string _stored_value_key; - feature_t _defaultValue; - vespalib::eval::ValueType _valueType; + vespalib::string _key; // 'foo' + vespalib::string _old_key; // '$foo' + vespalib::string _stored_value_key; // query.value.foo + vespalib::eval::ValueType _type; + feature_t _default_number_value; + vespalib::eval::Value::UP _default_object_value; + + fef::Property config_lookup(const fef::IIndexEnvironment &env) const; + fef::Property request_lookup(const fef::IQueryEnvironment &env) const; public: QueryBlueprint(); diff --git a/searchlib/src/vespa/searchlib/index/field_length_calculator.h b/searchlib/src/vespa/searchlib/index/field_length_calculator.h index 35a18b432f9..15d4c5ec285 100644 --- a/searchlib/src/vespa/searchlib/index/field_length_calculator.h +++ b/searchlib/src/vespa/searchlib/index/field_length_calculator.h @@ -15,7 +15,7 @@ namespace search::index { */ class FieldLengthCalculator { std::atomic<double> _average_field_length; - uint32_t _num_samples; // Capped by _max_num_samples + std::atomic<uint32_t> _num_samples; // Capped by _max_num_samples uint32_t _max_num_samples; public: @@ -39,7 +39,7 @@ public: } double get_average_field_length() const { return _average_field_length.load(std::memory_order_relaxed); } - uint32_t get_num_samples() const { return _num_samples; } + uint32_t get_num_samples() const { return _num_samples.load(std::memory_order_relaxed); } uint32_t get_max_num_samples() const { return _max_num_samples; } FieldLengthInfo get_info() const { @@ -47,10 +47,12 @@ public: } void add_field_length(uint32_t field_length) { - if (_num_samples < _max_num_samples) { - ++_num_samples; + auto num_samples = get_num_samples(); + if (num_samples < _max_num_samples) { + ++num_samples; + _num_samples.store(num_samples, std::memory_order_relaxed); } - _average_field_length.store((_average_field_length.load(std::memory_order_relaxed) * (_num_samples - 1) + field_length) / _num_samples, std::memory_order_relaxed); + _average_field_length.store((_average_field_length.load(std::memory_order_relaxed) * (num_samples - 1) + field_length) / num_samples, std::memory_order_relaxed); } }; diff --git a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp index 3293019e538..b851fc50518 100644 --- a/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/hitcollector.cpp @@ -37,7 +37,7 @@ HitCollector::sortHitsByDocId() HitCollector::HitCollector(uint32_t numDocs, uint32_t maxHitsSize) : _numDocs(numDocs), - _maxHitsSize(maxHitsSize), + _maxHitsSize(std::min(maxHitsSize, numDocs)), _maxDocIdVectorSize((numDocs + 31) / 32), _hits(), _hitsSortOrder(SortOrder::DOC_ID), @@ -53,7 +53,7 @@ HitCollector::HitCollector(uint32_t numDocs, } else { _collector = std::make_unique<DocIdCollector<false>>(*this); } - _hits.reserve(maxHitsSize); + _hits.reserve(_maxHitsSize); } HitCollector::~HitCollector() = default; diff --git a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h index 6782d8963d1..496226b59eb 100644 --- a/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h +++ b/searchlib/src/vespa/searchlib/queryeval/wand/weak_and_heap.h @@ -3,6 +3,7 @@ #include "wand_parts.h" #include <vespa/vespalib/util/priority_queue.h> +#include <atomic> #include <mutex> namespace search::queryeval { @@ -34,11 +35,11 @@ public: **/ uint32_t getScoresToTrack() const { return _scoresToTrack; } - score_t getMinScore() const { return _minScore; } + score_t getMinScore() const { return _minScore.load(std::memory_order_relaxed); } protected: - void setMinScore(score_t minScore) { _minScore = minScore; } + void setMinScore(score_t minScore) { _minScore.store(minScore, std::memory_order_relaxed); } private: - score_t _minScore; + std::atomic<score_t> _minScore; const uint32_t _scoresToTrack; }; diff --git a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.h b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.h index 6b5fd935b40..191a9670d91 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.h +++ b/searchlib/src/vespa/searchlib/transactionlog/trans_log_server_explorer.h @@ -2,7 +2,7 @@ #pragma once -#include <vespa/vespalib/net/state_explorer.h> +#include <vespa/vespalib/net/http/state_explorer.h> namespace search::transactionlog { diff --git a/searchlib/src/vespa/searchlib/uca/CMakeLists.txt b/searchlib/src/vespa/searchlib/uca/CMakeLists.txt index 6a4602381f4..75be1472a83 100644 --- a/searchlib/src/vespa/searchlib/uca/CMakeLists.txt +++ b/searchlib/src/vespa/searchlib/uca/CMakeLists.txt @@ -1,12 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -find_package(ICU 60.0 REQUIRED COMPONENTS uc i18n) -vespa_add_library(searchlib_searchlib_uca +vespa_add_library(searchlib_uca OBJECT SOURCES ucaconverter.cpp ucafunctionnode.cpp - INSTALL lib64 DEPENDS - searchlib - ICU::i18n - ICU::uc ) diff --git a/searchsummary/CMakeLists.txt b/searchsummary/CMakeLists.txt index ac392c6582c..d51e29dbd00 100644 --- a/searchsummary/CMakeLists.txt +++ b/searchsummary/CMakeLists.txt @@ -1,22 +1,14 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_define_module( DEPENDS - fastos vespalog vespalib - staging_vespalib - fnet configdefinitions - fastlib_fast document - config_cloudconfig - searchcommon - persistencetypes - metrics searchlib - juniper LIBS + src/vespa/juniper src/vespa/searchsummary src/vespa/searchsummary/config src/vespa/searchsummary/docsummary @@ -30,4 +22,5 @@ vespa_define_module( src/tests/docsummary/matched_elements_filter src/tests/docsummary/slime_summary src/tests/extractkeywords + src/tests/juniper ) diff --git a/juniper/src/test/.gitignore b/searchsummary/src/tests/juniper/.gitignore index 46b307da632..09d6225ca26 100644 --- a/juniper/src/test/.gitignore +++ b/searchsummary/src/tests/juniper/.gitignore @@ -14,3 +14,4 @@ juniper_auxTest_app juniper_matchobjectTest_app juniper_mcandTest_app juniper_queryparserTest_app +juniper_latintokenizertest_app diff --git a/juniper/src/test/CMakeLists.txt b/searchsummary/src/tests/juniper/CMakeLists.txt index c895d5d3766..77e7052a429 100644 --- a/juniper/src/test/CMakeLists.txt +++ b/searchsummary/src/tests/juniper/CMakeLists.txt @@ -1,13 +1,18 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_library(juniper_testsuite + SOURCES + test.cpp + DEPENDS +) + vespa_add_executable(juniper_mcandTest_app TEST SOURCES mcandTest.cpp mcandTestApp.cpp testenv.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary + juniper_testsuite ) vespa_add_test(NAME juniper_mcandTest_app COMMAND juniper_mcandTest_app) vespa_add_executable(juniper_queryparserTest_app TEST @@ -17,9 +22,8 @@ vespa_add_executable(juniper_queryparserTest_app TEST fakerewriter.cpp testenv.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary + juniper_testsuite ) vespa_add_test(NAME juniper_queryparserTest_app COMMAND juniper_queryparserTest_app) vespa_add_executable(juniper_matchobjectTest_app TEST @@ -29,27 +33,22 @@ vespa_add_executable(juniper_matchobjectTest_app TEST testenv.cpp fakerewriter.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary + juniper_testsuite ) vespa_add_test(NAME juniper_matchobjectTest_app COMMAND juniper_matchobjectTest_app) vespa_add_executable(juniper_appender_test_app TEST SOURCES appender_test.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary ) vespa_add_test(NAME juniper_appender_test_app COMMAND juniper_appender_test_app) vespa_add_executable(juniper_queryvisitor_test_app TEST SOURCES queryvisitor_test.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary ) vespa_add_test(NAME juniper_queryvisitor_test_app COMMAND juniper_queryvisitor_test_app) vespa_add_executable(juniper_auxTest_app TEST @@ -58,9 +57,8 @@ vespa_add_executable(juniper_auxTest_app TEST auxTestApp.cpp testenv.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary + juniper_testsuite ) vespa_add_test(NAME juniper_auxTest_app COMMAND juniper_auxTest_app) vespa_add_executable(juniper_SrcTestSuite_app TEST @@ -73,8 +71,14 @@ vespa_add_executable(juniper_SrcTestSuite_app TEST auxTest.cpp testenv.cpp DEPENDS - juniper - vespalib - fastlib_fast + searchsummary + juniper_testsuite ) vespa_add_test(NAME juniper_SrcTestSuite_app COMMAND juniper_SrcTestSuite_app) +vespa_add_executable(juniper_latintokenizertest_app TEST + SOURCES + latintokenizertest.cpp + DEPENDS + juniper_testsuite +) +vespa_add_test(NAME juniper_latintokenizertest_app NO_VALGRIND COMMAND juniper_latintokenizertest_app) diff --git a/juniper/src/test/SrcTestSuite.cpp b/searchsummary/src/tests/juniper/SrcTestSuite.cpp index 1fcaa6e29de..870c7b9f378 100644 --- a/juniper/src/test/SrcTestSuite.cpp +++ b/searchsummary/src/tests/juniper/SrcTestSuite.cpp @@ -1,12 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "testenv.h" +#include "suite.h" #include "mcandTest.h" #include "queryparserTest.h" #include "matchobjectTest.h" #include "auxTest.h" #include <vespa/vespalib/testkit/testapp.h> -#include <vespa/fastlib/testsuite/suite.h> /** * The SrcTestSuite class runs all the unit tests for the src module. * @@ -29,7 +29,7 @@ SrcTestSuite::SrcTestSuite() : } int main(int argc, char **argv) { - juniper::TestEnv te(argc, argv, TEST_PATH("../rpclient/testclient.rc").c_str()); + juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); SrcTestSuite suite; suite.Run(); long failures = suite.Report(); diff --git a/juniper/src/test/appender_test.cpp b/searchsummary/src/tests/juniper/appender_test.cpp index 65e876f1a35..65e876f1a35 100644 --- a/juniper/src/test/appender_test.cpp +++ b/searchsummary/src/tests/juniper/appender_test.cpp diff --git a/juniper/src/test/auxTest.cpp b/searchsummary/src/tests/juniper/auxTest.cpp index 15f5ad1749e..15f5ad1749e 100644 --- a/juniper/src/test/auxTest.cpp +++ b/searchsummary/src/tests/juniper/auxTest.cpp diff --git a/juniper/src/test/auxTest.h b/searchsummary/src/tests/juniper/auxTest.h index dd6d79e024a..9ff391911b3 100644 --- a/juniper/src/test/auxTest.h +++ b/searchsummary/src/tests/juniper/auxTest.h @@ -3,9 +3,9 @@ // Auxiliary tests for juniper - based on Juniper 1.x proximitytest.cpp -#include <map> -#include <vespa/fastlib/testsuite/test.h> #include "testenv.h" +#include "test.h" +#include <map> class AuxTest : public Test { diff --git a/juniper/src/test/auxTestApp.cpp b/searchsummary/src/tests/juniper/auxTestApp.cpp index 1c85796bb1f..5090e2d7dfc 100644 --- a/juniper/src/test/auxTestApp.cpp +++ b/searchsummary/src/tests/juniper/auxTestApp.cpp @@ -9,7 +9,7 @@ void Usage(char* s) int main(int argc, char **argv) { - juniper::TestEnv te(argc, argv, TEST_PATH("../rpclient/testclient.rc").c_str()); + juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); AuxTest pta; pta.SetStream(&std::cout); pta.Run(argc, argv); diff --git a/juniper/src/test/fakerewriter.cpp b/searchsummary/src/tests/juniper/fakerewriter.cpp index bbaf7079525..bbaf7079525 100644 --- a/juniper/src/test/fakerewriter.cpp +++ b/searchsummary/src/tests/juniper/fakerewriter.cpp diff --git a/juniper/src/test/fakerewriter.h b/searchsummary/src/tests/juniper/fakerewriter.h index e1e5de59feb..e1e5de59feb 100644 --- a/juniper/src/test/fakerewriter.h +++ b/searchsummary/src/tests/juniper/fakerewriter.h diff --git a/juniper/src/test/largeblockchinese.input.utf8 b/searchsummary/src/tests/juniper/largeblockchinese.input.utf8 index 9b85e8d06e6..9b85e8d06e6 100644 --- a/juniper/src/test/largeblockchinese.input.utf8 +++ b/searchsummary/src/tests/juniper/largeblockchinese.input.utf8 diff --git a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.cpp b/searchsummary/src/tests/juniper/latintokenizertest.cpp index 89273ab1ec0..89273ab1ec0 100644 --- a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.cpp +++ b/searchsummary/src/tests/juniper/latintokenizertest.cpp diff --git a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.h b/searchsummary/src/tests/juniper/latintokenizertest.h index 38dc08aa677..b4d113918ee 100644 --- a/fastlib/src/vespa/fastlib/text/tests/latintokenizertest.h +++ b/searchsummary/src/tests/juniper/latintokenizertest.h @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once -#include <vespa/fastlib/testsuite/test.h> -#include <vespa/fastlib/text/latintokenizer.h> +#include "test.h" +#include <vespa/juniper/latintokenizer.h> #include <vespa/vespalib/util/stringfmt.h> class Mapel_Pucntuation { diff --git a/juniper/src/test/matchobjectTest.cpp b/searchsummary/src/tests/juniper/matchobjectTest.cpp index 07e3cf84767..07e3cf84767 100644 --- a/juniper/src/test/matchobjectTest.cpp +++ b/searchsummary/src/tests/juniper/matchobjectTest.cpp diff --git a/juniper/src/test/matchobjectTest.h b/searchsummary/src/tests/juniper/matchobjectTest.h index 5bfd29a371f..9fdd3e4719f 100644 --- a/juniper/src/test/matchobjectTest.h +++ b/searchsummary/src/tests/juniper/matchobjectTest.h @@ -5,7 +5,7 @@ #pragma once #include "testenv.h" -#include <vespa/fastlib/testsuite/test.h> +#include "test.h" #include <map> /** diff --git a/searchsummary/src/tests/juniper/matchobjectTestApp.cpp b/searchsummary/src/tests/juniper/matchobjectTestApp.cpp new file mode 100644 index 00000000000..8bdebfe7207 --- /dev/null +++ b/searchsummary/src/tests/juniper/matchobjectTestApp.cpp @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "matchobjectTest.h" +#include "testenv.h" +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/juniper/wildcard_match.h> +#include <iostream> + +namespace { +void test(const char * word, const char * pattern, bool expect) { + EXPECT_EQUAL(expect, fast::util::wildcard_match(word, pattern)); +} +} + +void +test_wildcard() +{ + test("a", "b", false); + test("b", "b", true); + test("abc", "def", false); + test("def", "def", true); + test("def", "d?f", true); + test("def", "d?d", false); + test("def", "??d", false); + test("def", "d??", true); + test("abcdef", "a*e", false); + test("abcdef", "a*f", true); + test("abcdef", "a?c*f", true); + test("abcdef", "a?b*f", false); + test("abcdef", "a*b*f", true); + test("abcdef", "abc*", true); + test("abcdef", "*def", true); +} + +int main(int argc, char **argv) { + test_wildcard(); + juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); + MatchObjectTest test; + test.SetStream(&std::cout); + test.Run(argc, argv); + return (int)test.Report(); +} diff --git a/juniper/src/test/mcandTest.cpp b/searchsummary/src/tests/juniper/mcandTest.cpp index 5a465275a80..5a465275a80 100644 --- a/juniper/src/test/mcandTest.cpp +++ b/searchsummary/src/tests/juniper/mcandTest.cpp diff --git a/juniper/src/test/mcandTest.h b/searchsummary/src/tests/juniper/mcandTest.h index cdb01e91e3b..6eb8b4d66e5 100644 --- a/juniper/src/test/mcandTest.h +++ b/searchsummary/src/tests/juniper/mcandTest.h @@ -4,10 +4,10 @@ */ #pragma once -#include <map> -#include <vespa/fastlib/testsuite/test.h> #include "testenv.h" +#include "test.h" #include <vespa/juniper/mcand.h> +#include <map> /** * The MatchCandidateTest class holds diff --git a/juniper/src/test/mcandTestApp.cpp b/searchsummary/src/tests/juniper/mcandTestApp.cpp index 825dc61da06..7b1a15934d3 100644 --- a/juniper/src/test/mcandTestApp.cpp +++ b/searchsummary/src/tests/juniper/mcandTestApp.cpp @@ -4,7 +4,7 @@ #include <vespa/vespalib/testkit/testapp.h> int main(int argc, char **argv) { - juniper::TestEnv te(argc, argv, TEST_PATH("../rpclient/testclient.rc").c_str()); + juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); MatchCandidateTest test; test.SetStream(&std::cout); test.Run(argc, argv); diff --git a/juniper/src/test/partialutf8.input.utf8 b/searchsummary/src/tests/juniper/partialutf8.input.utf8 index df25fbb8c60..df25fbb8c60 100644 --- a/juniper/src/test/partialutf8.input.utf8 +++ b/searchsummary/src/tests/juniper/partialutf8.input.utf8 diff --git a/juniper/src/test/queryparserTest.cpp b/searchsummary/src/tests/juniper/queryparserTest.cpp index e8afbcc0cee..e8afbcc0cee 100644 --- a/juniper/src/test/queryparserTest.cpp +++ b/searchsummary/src/tests/juniper/queryparserTest.cpp diff --git a/juniper/src/test/queryparserTest.h b/searchsummary/src/tests/juniper/queryparserTest.h index 7dc4dda63fa..803fbd4c999 100644 --- a/juniper/src/test/queryparserTest.h +++ b/searchsummary/src/tests/juniper/queryparserTest.h @@ -5,9 +5,9 @@ #pragma once #include "testenv.h" +#include "test.h" #include <vespa/juniper/queryparser.h> #include <vespa/juniper/rewriter.h> -#include <vespa/fastlib/testsuite/test.h> #include <map> /** diff --git a/juniper/src/test/queryparserTestApp.cpp b/searchsummary/src/tests/juniper/queryparserTestApp.cpp index a2c551a4730..c34f0b77ae9 100644 --- a/juniper/src/test/queryparserTestApp.cpp +++ b/searchsummary/src/tests/juniper/queryparserTestApp.cpp @@ -6,7 +6,7 @@ int main(int argc, char **argv) { - juniper::TestEnv te(argc, argv, TEST_PATH("../rpclient/testclient.rc").c_str()); + juniper::TestEnv te(argc, argv, TEST_PATH("./testclient.rc").c_str()); QueryParserTest test; test.SetStream(&std::cout); test.Run(argc, argv); diff --git a/juniper/src/test/queryvisitor_test.cpp b/searchsummary/src/tests/juniper/queryvisitor_test.cpp index 0b99bf6583c..0b99bf6583c 100644 --- a/juniper/src/test/queryvisitor_test.cpp +++ b/searchsummary/src/tests/juniper/queryvisitor_test.cpp diff --git a/fastlib/src/vespa/fastlib/testsuite/suite.h b/searchsummary/src/tests/juniper/suite.h index fea685731ae..fea685731ae 100644 --- a/fastlib/src/vespa/fastlib/testsuite/suite.h +++ b/searchsummary/src/tests/juniper/suite.h diff --git a/fastlib/src/vespa/fastlib/testsuite/test.cpp b/searchsummary/src/tests/juniper/test.cpp index 18930b1bca2..18930b1bca2 100644 --- a/fastlib/src/vespa/fastlib/testsuite/test.cpp +++ b/searchsummary/src/tests/juniper/test.cpp diff --git a/fastlib/src/vespa/fastlib/testsuite/test.h b/searchsummary/src/tests/juniper/test.h index 1388c3ba812..1388c3ba812 100644 --- a/fastlib/src/vespa/fastlib/testsuite/test.h +++ b/searchsummary/src/tests/juniper/test.h diff --git a/juniper/src/rpclient/testclient.rc b/searchsummary/src/tests/juniper/testclient.rc index d04262c364c..d04262c364c 100644 --- a/juniper/src/rpclient/testclient.rc +++ b/searchsummary/src/tests/juniper/testclient.rc diff --git a/juniper/src/test/testenv.cpp b/searchsummary/src/tests/juniper/testenv.cpp index 769c24b829c..769c24b829c 100644 --- a/juniper/src/test/testenv.cpp +++ b/searchsummary/src/tests/juniper/testenv.cpp diff --git a/juniper/src/test/testenv.h b/searchsummary/src/tests/juniper/testenv.h index a43f4a11bec..a43f4a11bec 100644 --- a/juniper/src/test/testenv.h +++ b/searchsummary/src/tests/juniper/testenv.h diff --git a/juniper/src/vespa/juniper/.gitignore b/searchsummary/src/vespa/juniper/.gitignore index d4d35b71c0f..d4d35b71c0f 100644 --- a/juniper/src/vespa/juniper/.gitignore +++ b/searchsummary/src/vespa/juniper/.gitignore diff --git a/juniper/src/vespa/juniper/CMakeLists.txt b/searchsummary/src/vespa/juniper/CMakeLists.txt index 5420aa203ea..3d6b72ef511 100644 --- a/juniper/src/vespa/juniper/CMakeLists.txt +++ b/searchsummary/src/vespa/juniper/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(juniper +vespa_add_library(searchsummary_juniper OBJECT SOURCES Matcher.cpp appender.cpp @@ -25,6 +25,5 @@ vespa_add_library(juniper expcache.cpp reducematcher.cpp specialtokenregistry.cpp - INSTALL lib64 DEPENDS ) diff --git a/juniper/src/vespa/juniper/IJuniperProperties.h b/searchsummary/src/vespa/juniper/IJuniperProperties.h index 63ada17684c..63ada17684c 100644 --- a/juniper/src/vespa/juniper/IJuniperProperties.h +++ b/searchsummary/src/vespa/juniper/IJuniperProperties.h diff --git a/juniper/src/vespa/juniper/ITokenProcessor.h b/searchsummary/src/vespa/juniper/ITokenProcessor.h index fbb9a93075c..fbb9a93075c 100644 --- a/juniper/src/vespa/juniper/ITokenProcessor.h +++ b/searchsummary/src/vespa/juniper/ITokenProcessor.h diff --git a/juniper/src/vespa/juniper/Matcher.cpp b/searchsummary/src/vespa/juniper/Matcher.cpp index e286068038b..e286068038b 100644 --- a/juniper/src/vespa/juniper/Matcher.cpp +++ b/searchsummary/src/vespa/juniper/Matcher.cpp diff --git a/juniper/src/vespa/juniper/Matcher.h b/searchsummary/src/vespa/juniper/Matcher.h index 5564537ae4a..5564537ae4a 100644 --- a/juniper/src/vespa/juniper/Matcher.h +++ b/searchsummary/src/vespa/juniper/Matcher.h diff --git a/juniper/src/vespa/juniper/SummaryConfig.cpp b/searchsummary/src/vespa/juniper/SummaryConfig.cpp index 5b009ea5cd2..5b009ea5cd2 100644 --- a/juniper/src/vespa/juniper/SummaryConfig.cpp +++ b/searchsummary/src/vespa/juniper/SummaryConfig.cpp diff --git a/juniper/src/vespa/juniper/SummaryConfig.h b/searchsummary/src/vespa/juniper/SummaryConfig.h index fdf4cdcaae1..fdf4cdcaae1 100644 --- a/juniper/src/vespa/juniper/SummaryConfig.h +++ b/searchsummary/src/vespa/juniper/SummaryConfig.h diff --git a/juniper/src/vespa/juniper/appender.cpp b/searchsummary/src/vespa/juniper/appender.cpp index 4d55f62a27a..4d55f62a27a 100644 --- a/juniper/src/vespa/juniper/appender.cpp +++ b/searchsummary/src/vespa/juniper/appender.cpp diff --git a/juniper/src/vespa/juniper/appender.h b/searchsummary/src/vespa/juniper/appender.h index 760fd18feca..760fd18feca 100644 --- a/juniper/src/vespa/juniper/appender.h +++ b/searchsummary/src/vespa/juniper/appender.h diff --git a/juniper/src/vespa/juniper/charutil.h b/searchsummary/src/vespa/juniper/charutil.h index 859bb6e6ed9..859bb6e6ed9 100644 --- a/juniper/src/vespa/juniper/charutil.h +++ b/searchsummary/src/vespa/juniper/charutil.h diff --git a/juniper/src/vespa/juniper/config.cpp b/searchsummary/src/vespa/juniper/config.cpp index b9213bb21f1..a82a8d74b8a 100644 --- a/juniper/src/vespa/juniper/config.cpp +++ b/searchsummary/src/vespa/juniper/config.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "config.h" -#include "IJuniperProperties.h" #include "rpinterface.h" #include "juniperdebug.h" #define _NEED_SUMMARY_CONFIG_IMPL @@ -14,7 +13,7 @@ namespace juniper Config::Config(const char* config_name, Juniper & juniper) : _docsumparams(), _matcherparams(), - _sumconf(NULL), + _sumconf(nullptr), _config_name(config_name), _juniper(juniper) { @@ -51,7 +50,7 @@ Config::Config(const char* config_name, Juniper & juniper) : .SetMaxMatches(max_matches) .SetSurroundMax(surround_max) .SetFallback(fallback); - _matcherparams.SetWantGlobalRank(true) + _matcherparams .SetStemMinLength(stem_min).SetStemMaxExtend(stem_extend) .SetMatchWindowSize(match_winsize) .SetMaxMatchCandidates(max_match_candidates) diff --git a/juniper/src/vespa/juniper/config.h b/searchsummary/src/vespa/juniper/config.h index a9dabdd91d1..a9dabdd91d1 100644 --- a/juniper/src/vespa/juniper/config.h +++ b/searchsummary/src/vespa/juniper/config.h diff --git a/juniper/src/vespa/juniper/dpinterface.cpp b/searchsummary/src/vespa/juniper/dpinterface.cpp index 7b7c5aa7120..7b7c5aa7120 100644 --- a/juniper/src/vespa/juniper/dpinterface.cpp +++ b/searchsummary/src/vespa/juniper/dpinterface.cpp diff --git a/juniper/src/vespa/juniper/dpinterface.h b/searchsummary/src/vespa/juniper/dpinterface.h index 8f538cee05d..8f538cee05d 100644 --- a/juniper/src/vespa/juniper/dpinterface.h +++ b/searchsummary/src/vespa/juniper/dpinterface.h diff --git a/juniper/src/vespa/juniper/expcache.cpp b/searchsummary/src/vespa/juniper/expcache.cpp index 7436054932b..7436054932b 100644 --- a/juniper/src/vespa/juniper/expcache.cpp +++ b/searchsummary/src/vespa/juniper/expcache.cpp diff --git a/juniper/src/vespa/juniper/expcache.h b/searchsummary/src/vespa/juniper/expcache.h index 5f16397d69a..5f16397d69a 100644 --- a/juniper/src/vespa/juniper/expcache.h +++ b/searchsummary/src/vespa/juniper/expcache.h diff --git a/juniper/src/vespa/juniper/foreach_utils.h b/searchsummary/src/vespa/juniper/foreach_utils.h index ebbf1f41049..ebbf1f41049 100644 --- a/juniper/src/vespa/juniper/foreach_utils.h +++ b/searchsummary/src/vespa/juniper/foreach_utils.h diff --git a/juniper/src/vespa/juniper/hashbase.h b/searchsummary/src/vespa/juniper/hashbase.h index 2cdfc02800f..2cdfc02800f 100644 --- a/juniper/src/vespa/juniper/hashbase.h +++ b/searchsummary/src/vespa/juniper/hashbase.h diff --git a/juniper/src/vespa/juniper/juniperdebug.h b/searchsummary/src/vespa/juniper/juniperdebug.h index cf6a3c971f8..cf6a3c971f8 100644 --- a/juniper/src/vespa/juniper/juniperdebug.h +++ b/searchsummary/src/vespa/juniper/juniperdebug.h diff --git a/juniper/src/vespa/juniper/juniperparams.cpp b/searchsummary/src/vespa/juniper/juniperparams.cpp index e600c23f7c4..4f25b2446ad 100644 --- a/juniper/src/vespa/juniper/juniperparams.cpp +++ b/searchsummary/src/vespa/juniper/juniperparams.cpp @@ -1,7 +1,5 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "juniperdebug.h" #include "juniperparams.h" -#include "Matcher.h" #include <cstring> // DocsumParams implementation: @@ -9,7 +7,7 @@ DocsumParams::DocsumParams() : _enabled(false), _length(256), _min_length(128), _max_matches(3), - _surround_max(80), _space_chars(""), _fallback(FALLBACK_NONE) + _surround_max(80), _fallback(FALLBACK_NONE) { } DocsumParams& DocsumParams::SetEnabled(bool en) @@ -42,12 +40,6 @@ DocsumParams& DocsumParams::SetSurroundMax(size_t length) return *this; } -DocsumParams& DocsumParams::SetSpaceChars(const char* spacechars) -{ - _space_chars = spacechars; - return *this; -} - DocsumParams& DocsumParams::SetFallback(const char* fallback) { if (strcmp("prefix", fallback) == 0) { @@ -63,7 +55,6 @@ size_t DocsumParams::MinLength() const { return _min_length; } size_t DocsumParams::MaxMatches() const { return _max_matches; } size_t DocsumParams::SurroundMax() const { return _surround_max; } bool DocsumParams::Enabled() const { return _enabled; } -const char* DocsumParams::SpaceChars() const { return _space_chars.c_str(); } int DocsumParams::Fallback() const { return _fallback; } // MatcherParams implementation: @@ -71,54 +62,26 @@ int DocsumParams::Fallback() const { return _fallback; } MatcherParams::MatcherParams() : - _prefix_extend_length(3), - _prefix_min_length(5), _match_winsize(200), _match_winsize_fallback_multiplier(10.0), _max_match_candidates(1000), - _want_global_rank(false), _stem_min(0), _stem_extend(0), _wordfolder(NULL), _proximity_factor(1.0) { } -MatcherParams& MatcherParams::SetPrefixExtendLength(size_t extend_length) -{ - _prefix_extend_length = extend_length; - return *this; -} - -MatcherParams& MatcherParams::SetPrefixMinLength(size_t min_length) -{ - _prefix_min_length = min_length; - return *this; -} - - MatcherParams& MatcherParams::SetMatchWindowSize(size_t winsize) { _match_winsize = winsize; return *this; } -MatcherParams& MatcherParams::SetMatchWindowSizeFallbackMultiplier(double winsize) -{ - _match_winsize_fallback_multiplier = winsize; - return *this; -} - MatcherParams& MatcherParams::SetMaxMatchCandidates(size_t max_match_candidates) { _max_match_candidates = max_match_candidates; return *this; } -MatcherParams& MatcherParams::SetWantGlobalRank(bool global_rank) -{ - _want_global_rank = global_rank; - return *this; -} - MatcherParams& MatcherParams::SetStemMinLength(size_t stem_min) { _stem_min = stem_min; @@ -132,12 +95,9 @@ MatcherParams& MatcherParams::SetStemMaxExtend(size_t stem_extend) return *this; } -size_t MatcherParams::PrefixExtendLength() const { return _prefix_extend_length; } -size_t MatcherParams::PrefixMinLength() const { return _prefix_min_length; } size_t MatcherParams::MatchWindowSize() const { return _match_winsize; } double MatcherParams::MatchWindowSizeFallbackMultiplier() const { return _match_winsize_fallback_multiplier; } size_t MatcherParams::MaxMatchCandidates() const { return _max_match_candidates; } -bool MatcherParams::WantGlobalRank() const { return _want_global_rank; } size_t MatcherParams::StemMinLength() const { return _stem_min; } size_t MatcherParams::StemMaxExtend() const { return _stem_extend; } diff --git a/juniper/src/vespa/juniper/juniperparams.h b/searchsummary/src/vespa/juniper/juniperparams.h index 44980ce8b43..f4f17779f2d 100644 --- a/juniper/src/vespa/juniper/juniperparams.h +++ b/searchsummary/src/vespa/juniper/juniperparams.h @@ -31,9 +31,6 @@ public: DocsumParams& SetSurroundMax(size_t length); size_t SurroundMax() const; - DocsumParams& SetSpaceChars(const char* spacechars); - const char* SpaceChars() const; - DocsumParams& SetFallback(const char* fallback); int Fallback() const; @@ -43,7 +40,6 @@ private: size_t _min_length; size_t _max_matches; size_t _surround_max; - std::string _space_chars; int _fallback; }; @@ -52,25 +48,17 @@ class MatcherParams { public: MatcherParams(); - - MatcherParams& SetPrefixExtendLength(size_t extend_length); - size_t PrefixExtendLength() const; - - MatcherParams& SetPrefixMinLength(size_t min_length); - size_t PrefixMinLength() const; + MatcherParams(MatcherParams &) = delete; + MatcherParams &operator=(MatcherParams &) = delete; MatcherParams& SetMatchWindowSize(size_t winsize); size_t MatchWindowSize() const; - MatcherParams& SetMatchWindowSizeFallbackMultiplier(double winsize); double MatchWindowSizeFallbackMultiplier() const; MatcherParams& SetMaxMatchCandidates(size_t max_match_candidates); size_t MaxMatchCandidates() const; - MatcherParams& SetWantGlobalRank(bool global_rank); - bool WantGlobalRank() const; - MatcherParams& SetStemMinLength(size_t stem_min); size_t StemMinLength() const; @@ -84,19 +72,13 @@ public: double ProximityFactor(); private: - size_t _prefix_extend_length; - size_t _prefix_min_length; size_t _match_winsize; double _match_winsize_fallback_multiplier; size_t _max_match_candidates; - bool _want_global_rank; size_t _stem_min; size_t _stem_extend; Fast_WordFolder* _wordfolder; // The wordfolder object needed as 1st parameter to folderfun double _proximity_factor; - - MatcherParams(MatcherParams &); - MatcherParams &operator=(MatcherParams &); }; diff --git a/juniper/src/vespa/juniper/keyocc.cpp b/searchsummary/src/vespa/juniper/keyocc.cpp index 7214bc72fe9..7214bc72fe9 100644 --- a/juniper/src/vespa/juniper/keyocc.cpp +++ b/searchsummary/src/vespa/juniper/keyocc.cpp diff --git a/juniper/src/vespa/juniper/keyocc.h b/searchsummary/src/vespa/juniper/keyocc.h index 8c79e51e3a3..8c79e51e3a3 100644 --- a/juniper/src/vespa/juniper/keyocc.h +++ b/searchsummary/src/vespa/juniper/keyocc.h diff --git a/fastlib/src/vespa/fastlib/text/latintokenizer.h b/searchsummary/src/vespa/juniper/latintokenizer.h index be5d98054d9..7a98d780c56 100644 --- a/fastlib/src/vespa/fastlib/text/latintokenizer.h +++ b/searchsummary/src/vespa/juniper/latintokenizer.h @@ -34,8 +34,8 @@ #pragma once -#include <ctype.h> -#include <string.h> +#include <cctype> +#include <cstring> /** ***************************************************************************** diff --git a/juniper/src/vespa/juniper/matchelem.cpp b/searchsummary/src/vespa/juniper/matchelem.cpp index 27c4c9516e9..27c4c9516e9 100644 --- a/juniper/src/vespa/juniper/matchelem.cpp +++ b/searchsummary/src/vespa/juniper/matchelem.cpp diff --git a/juniper/src/vespa/juniper/matchelem.h b/searchsummary/src/vespa/juniper/matchelem.h index 7948eabb8ed..7948eabb8ed 100644 --- a/juniper/src/vespa/juniper/matchelem.h +++ b/searchsummary/src/vespa/juniper/matchelem.h diff --git a/juniper/src/vespa/juniper/matchobject.cpp b/searchsummary/src/vespa/juniper/matchobject.cpp index d6657250f80..376f970d73b 100644 --- a/juniper/src/vespa/juniper/matchobject.cpp +++ b/searchsummary/src/vespa/juniper/matchobject.cpp @@ -5,7 +5,7 @@ #include "juniperdebug.h" #include "result.h" #include "charutil.h" -#include <vespa/fastlib/util/wildcard_match.h> +#include "wildcard_match.h" #include <stack> #include <vespa/log/log.h> LOG_SETUP(".juniper.matchobject"); diff --git a/juniper/src/vespa/juniper/matchobject.h b/searchsummary/src/vespa/juniper/matchobject.h index 3278448e0f7..3278448e0f7 100644 --- a/juniper/src/vespa/juniper/matchobject.h +++ b/searchsummary/src/vespa/juniper/matchobject.h diff --git a/juniper/src/vespa/juniper/mcand.cpp b/searchsummary/src/vespa/juniper/mcand.cpp index f67788d59ed..f67788d59ed 100644 --- a/juniper/src/vespa/juniper/mcand.cpp +++ b/searchsummary/src/vespa/juniper/mcand.cpp diff --git a/juniper/src/vespa/juniper/mcand.h b/searchsummary/src/vespa/juniper/mcand.h index a09a18267c1..a09a18267c1 100644 --- a/juniper/src/vespa/juniper/mcand.h +++ b/searchsummary/src/vespa/juniper/mcand.h diff --git a/juniper/src/vespa/juniper/propreader.cpp b/searchsummary/src/vespa/juniper/propreader.cpp index 99a6e580126..99a6e580126 100644 --- a/juniper/src/vespa/juniper/propreader.cpp +++ b/searchsummary/src/vespa/juniper/propreader.cpp diff --git a/juniper/src/vespa/juniper/propreader.h b/searchsummary/src/vespa/juniper/propreader.h index fbc6f53bfb1..fbc6f53bfb1 100644 --- a/juniper/src/vespa/juniper/propreader.h +++ b/searchsummary/src/vespa/juniper/propreader.h diff --git a/juniper/src/vespa/juniper/query.h b/searchsummary/src/vespa/juniper/query.h index 8208b234d6b..8208b234d6b 100644 --- a/juniper/src/vespa/juniper/query.h +++ b/searchsummary/src/vespa/juniper/query.h diff --git a/juniper/src/vespa/juniper/queryhandle.cpp b/searchsummary/src/vespa/juniper/queryhandle.cpp index ab2e24970fd..ab2e24970fd 100644 --- a/juniper/src/vespa/juniper/queryhandle.cpp +++ b/searchsummary/src/vespa/juniper/queryhandle.cpp diff --git a/juniper/src/vespa/juniper/queryhandle.h b/searchsummary/src/vespa/juniper/queryhandle.h index d0b266e99ce..d0b266e99ce 100644 --- a/juniper/src/vespa/juniper/queryhandle.h +++ b/searchsummary/src/vespa/juniper/queryhandle.h diff --git a/juniper/src/vespa/juniper/querymodifier.cpp b/searchsummary/src/vespa/juniper/querymodifier.cpp index 797617b605a..797617b605a 100644 --- a/juniper/src/vespa/juniper/querymodifier.cpp +++ b/searchsummary/src/vespa/juniper/querymodifier.cpp diff --git a/juniper/src/vespa/juniper/querymodifier.h b/searchsummary/src/vespa/juniper/querymodifier.h index ec0d20778fb..ec0d20778fb 100644 --- a/juniper/src/vespa/juniper/querymodifier.h +++ b/searchsummary/src/vespa/juniper/querymodifier.h diff --git a/juniper/src/vespa/juniper/querynode.cpp b/searchsummary/src/vespa/juniper/querynode.cpp index 1a7a80333c7..1a7a80333c7 100644 --- a/juniper/src/vespa/juniper/querynode.cpp +++ b/searchsummary/src/vespa/juniper/querynode.cpp diff --git a/juniper/src/vespa/juniper/querynode.h b/searchsummary/src/vespa/juniper/querynode.h index c6fecaf1b9f..c6fecaf1b9f 100644 --- a/juniper/src/vespa/juniper/querynode.h +++ b/searchsummary/src/vespa/juniper/querynode.h diff --git a/juniper/src/vespa/juniper/queryparser.cpp b/searchsummary/src/vespa/juniper/queryparser.cpp index 9b8169bee44..9b8169bee44 100644 --- a/juniper/src/vespa/juniper/queryparser.cpp +++ b/searchsummary/src/vespa/juniper/queryparser.cpp diff --git a/juniper/src/vespa/juniper/queryparser.h b/searchsummary/src/vespa/juniper/queryparser.h index 5715daa3661..9c596892e31 100644 --- a/juniper/src/vespa/juniper/queryparser.h +++ b/searchsummary/src/vespa/juniper/queryparser.h @@ -1,13 +1,12 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once - /* Simple prefix syntax advanced query parser for Juniper debug/testing */ #include "query.h" +#include "latintokenizer.h" #include <map> #include <string> -#include <vespa/fastlib/text/latintokenizer.h> namespace juniper { diff --git a/juniper/src/vespa/juniper/queryvisitor.cpp b/searchsummary/src/vespa/juniper/queryvisitor.cpp index dcd716c0c0f..dcd716c0c0f 100644 --- a/juniper/src/vespa/juniper/queryvisitor.cpp +++ b/searchsummary/src/vespa/juniper/queryvisitor.cpp diff --git a/juniper/src/vespa/juniper/queryvisitor.h b/searchsummary/src/vespa/juniper/queryvisitor.h index 7c82d3fa50a..7c82d3fa50a 100644 --- a/juniper/src/vespa/juniper/queryvisitor.h +++ b/searchsummary/src/vespa/juniper/queryvisitor.h diff --git a/juniper/src/vespa/juniper/reducematcher.cpp b/searchsummary/src/vespa/juniper/reducematcher.cpp index 642daae9773..642daae9773 100644 --- a/juniper/src/vespa/juniper/reducematcher.cpp +++ b/searchsummary/src/vespa/juniper/reducematcher.cpp diff --git a/juniper/src/vespa/juniper/reducematcher.h b/searchsummary/src/vespa/juniper/reducematcher.h index cfdc252b17e..cfdc252b17e 100644 --- a/juniper/src/vespa/juniper/reducematcher.h +++ b/searchsummary/src/vespa/juniper/reducematcher.h diff --git a/juniper/src/vespa/juniper/result.cpp b/searchsummary/src/vespa/juniper/result.cpp index 653e692e015..15ad9aa2a98 100644 --- a/juniper/src/vespa/juniper/result.cpp +++ b/searchsummary/src/vespa/juniper/result.cpp @@ -80,12 +80,12 @@ Result::Result(Config* config, QueryHandle* qhandle, } /* Create the new pipeline */ - _tokenizer.reset(new JuniperTokenizer(wordfolder, NULL, 0, NULL)); + _tokenizer = std::make_unique<JuniperTokenizer>(wordfolder, nullptr, 0, nullptr, nullptr); - _matcher.reset(new Matcher(this)); + _matcher = std::make_unique<Matcher>(this); _matcher->SetProximityFactor(mp.ProximityFactor()); - _registry.reset(new SpecialTokenRegistry(_matcher->getQuery())); + _registry = std::make_unique<SpecialTokenRegistry>(_matcher->getQuery()); if (qhandle->_log_mask) _matcher->set_log(qhandle->_log_mask); diff --git a/juniper/src/vespa/juniper/result.h b/searchsummary/src/vespa/juniper/result.h index f0dcf3d4335..f0dcf3d4335 100644 --- a/juniper/src/vespa/juniper/result.h +++ b/searchsummary/src/vespa/juniper/result.h diff --git a/juniper/src/vespa/juniper/rewriter.h b/searchsummary/src/vespa/juniper/rewriter.h index 92542da5acc..92542da5acc 100644 --- a/juniper/src/vespa/juniper/rewriter.h +++ b/searchsummary/src/vespa/juniper/rewriter.h diff --git a/juniper/src/vespa/juniper/rpinterface.cpp b/searchsummary/src/vespa/juniper/rpinterface.cpp index 75a441fb957..f9e91073a9b 100644 --- a/juniper/src/vespa/juniper/rpinterface.cpp +++ b/searchsummary/src/vespa/juniper/rpinterface.cpp @@ -94,12 +94,6 @@ void Juniper::FlushRewriters() _modifier->FlushRewriters(); } -void ReleaseConfig(Config*& config) -{ - delete config; - config = NULL; -} - void ReleaseQueryHandle(QueryHandle*& handle) { diff --git a/juniper/src/vespa/juniper/rpinterface.h b/searchsummary/src/vespa/juniper/rpinterface.h index 6cda324ae5c..6cda324ae5c 100644 --- a/juniper/src/vespa/juniper/rpinterface.h +++ b/searchsummary/src/vespa/juniper/rpinterface.h diff --git a/juniper/src/vespa/juniper/simplemap.h b/searchsummary/src/vespa/juniper/simplemap.h index 079637de231..079637de231 100644 --- a/juniper/src/vespa/juniper/simplemap.h +++ b/searchsummary/src/vespa/juniper/simplemap.h diff --git a/juniper/src/vespa/juniper/specialtokenregistry.cpp b/searchsummary/src/vespa/juniper/specialtokenregistry.cpp index cda8bc0fcd7..cda8bc0fcd7 100644 --- a/juniper/src/vespa/juniper/specialtokenregistry.cpp +++ b/searchsummary/src/vespa/juniper/specialtokenregistry.cpp diff --git a/juniper/src/vespa/juniper/specialtokenregistry.h b/searchsummary/src/vespa/juniper/specialtokenregistry.h index 2240dcf65b0..2240dcf65b0 100644 --- a/juniper/src/vespa/juniper/specialtokenregistry.h +++ b/searchsummary/src/vespa/juniper/specialtokenregistry.h diff --git a/juniper/src/vespa/juniper/stringmap.cpp b/searchsummary/src/vespa/juniper/stringmap.cpp index 3cb9e9b62e0..3cb9e9b62e0 100644 --- a/juniper/src/vespa/juniper/stringmap.cpp +++ b/searchsummary/src/vespa/juniper/stringmap.cpp diff --git a/juniper/src/vespa/juniper/stringmap.h b/searchsummary/src/vespa/juniper/stringmap.h index 4b6265695ad..4b6265695ad 100644 --- a/juniper/src/vespa/juniper/stringmap.h +++ b/searchsummary/src/vespa/juniper/stringmap.h diff --git a/juniper/src/vespa/juniper/sumdesc.cpp b/searchsummary/src/vespa/juniper/sumdesc.cpp index fcee1eb605f..fcee1eb605f 100644 --- a/juniper/src/vespa/juniper/sumdesc.cpp +++ b/searchsummary/src/vespa/juniper/sumdesc.cpp diff --git a/juniper/src/vespa/juniper/sumdesc.h b/searchsummary/src/vespa/juniper/sumdesc.h index d91bf160e04..d91bf160e04 100644 --- a/juniper/src/vespa/juniper/sumdesc.h +++ b/searchsummary/src/vespa/juniper/sumdesc.h diff --git a/juniper/src/vespa/juniper/tokenizer.cpp b/searchsummary/src/vespa/juniper/tokenizer.cpp index db6e1ecfccd..db6e1ecfccd 100644 --- a/juniper/src/vespa/juniper/tokenizer.cpp +++ b/searchsummary/src/vespa/juniper/tokenizer.cpp diff --git a/juniper/src/vespa/juniper/tokenizer.h b/searchsummary/src/vespa/juniper/tokenizer.h index 34ed1dba5bb..34ed1dba5bb 100644 --- a/juniper/src/vespa/juniper/tokenizer.h +++ b/searchsummary/src/vespa/juniper/tokenizer.h diff --git a/fastlib/src/vespa/fastlib/util/wildcard_match.h b/searchsummary/src/vespa/juniper/wildcard_match.h index 2cde4364693..2cde4364693 100644 --- a/fastlib/src/vespa/fastlib/util/wildcard_match.h +++ b/searchsummary/src/vespa/juniper/wildcard_match.h diff --git a/searchsummary/src/vespa/searchsummary/CMakeLists.txt b/searchsummary/src/vespa/searchsummary/CMakeLists.txt index 917e8b017da..6ec91622980 100644 --- a/searchsummary/src/vespa/searchsummary/CMakeLists.txt +++ b/searchsummary/src/vespa/searchsummary/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_library(searchsummary SOURCES $<TARGET_OBJECTS:searchsummary_config> $<TARGET_OBJECTS:searchsummary_docsummary> + $<TARGET_OBJECTS:searchsummary_juniper> INSTALL lib64 DEPENDS z diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp index cef0ba5793b..cf3def60f4e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.cpp @@ -1,7 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "juniperproperties.h" -#include <vespa/searchcommon/config/subscriptionproxyng.h> #include <vespa/vespalib/util/stringfmt.h> using vespa::config::search::summary::JuniperrcConfig; @@ -22,9 +21,7 @@ JuniperProperties::JuniperProperties(const JuniperrcConfig &cfg) : configure(cfg); } -JuniperProperties::~JuniperProperties() { - // empty -} +JuniperProperties::~JuniperProperties() = default; void JuniperProperties::reset() @@ -87,13 +84,6 @@ JuniperProperties::configure(const JuniperrcConfig &cfg) } } -void -JuniperProperties::subscribe(const char *configId) -{ - SubscriptionProxyNg<JuniperProperties, JuniperrcConfig> subscriber(*this, &JuniperProperties::configure); - subscriber.subscribe(configId); -} - const char * JuniperProperties::GetProperty(const char *name, const char *def) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h index 856d564acc6..6b1613b6c07 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniperproperties.h @@ -34,16 +34,6 @@ public: virtual ~JuniperProperties(); /** - * This method subscribes to config from the given configuration id. This does the necessary mapping from - * user-friendly configuration parameters to juniper specific properties. Note that no exceptions thrown by the - * configuration framework are caught in this method. Please refer to the config framework for details on what to - * expect. - * - * @param configId The config id to subscribe to. - */ - void subscribe(const char *configId); - - /** * Implements configure callback for config subscription. * * @param cfg The configuration object. @@ -51,7 +41,7 @@ public: void configure(const vespa::config::search::summary::JuniperrcConfig &cfg); // Inherit doc from IJuniperProperties. - const char *GetProperty(const char *name, const char *def = NULL) override; + const char *GetProperty(const char *name, const char *def) override; }; } diff --git a/security-tools/CMakeLists.txt b/security-tools/CMakeLists.txt index 4e426d43160..8341f57b9cc 100644 --- a/security-tools/CMakeLists.txt +++ b/security-tools/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(security-tools) +install_jar(security-tools-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-security-env vespa-security-env bin) vespa_install_script(src/main/sh/vespa-curl-wrapper vespa-curl-wrapper libexec/vespa) diff --git a/security-utils/CMakeLists.txt b/security-utils/CMakeLists.txt index 1ca28662d4e..b7700f45876 100644 --- a/security-utils/CMakeLists.txt +++ b/security-utils/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact(security-utils) +install_jar(security-utils.jar) diff --git a/service-monitor/CMakeLists.txt b/service-monitor/CMakeLists.txt index 8d505e5c00b..4be67e8f7b2 100644 --- a/service-monitor/CMakeLists.txt +++ b/service-monitor/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(service-monitor) +install_jar(service-monitor-jar-with-dependencies.jar) diff --git a/slobrok/CMakeLists.txt b/slobrok/CMakeLists.txt index 9a8f73c6e86..332e1e90282 100644 --- a/slobrok/CMakeLists.txt +++ b/slobrok/CMakeLists.txt @@ -5,7 +5,6 @@ vespa_define_module( vespalib fnet configdefinitions - staging_vespalib LIBS src/vespa/slobrok diff --git a/slobrok/src/vespa/slobrok/sbmirror.cpp b/slobrok/src/vespa/slobrok/sbmirror.cpp index 9207eb3f56f..62d288d40dc 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.cpp +++ b/slobrok/src/vespa/slobrok/sbmirror.cpp @@ -217,8 +217,8 @@ MirrorAPI::handleReconfig() bool MirrorAPI::handleReqDone() { - if (_reqDone) { - _reqDone = false; + if (_reqDone.load(std::memory_order_relaxed)) { + _reqDone.store(false, std::memory_order_relaxed); _reqPending = false; bool reconn = _req->IsError() ? true : handleIncrementalFetch(); @@ -321,9 +321,9 @@ MirrorAPI::PerformTask() void MirrorAPI::RequestDone(FRT_RPCRequest *req) { - LOG_ASSERT(req == _req && !_reqDone); + LOG_ASSERT(req == _req && !_reqDone.load(std::memory_order_relaxed)); (void) req; - _reqDone = true; + _reqDone.store(true, std::memory_order_relaxed); ScheduleNow(); } diff --git a/slobrok/src/vespa/slobrok/sbmirror.h b/slobrok/src/vespa/slobrok/sbmirror.h index 58258ecf023..d13a52f2b49 100644 --- a/slobrok/src/vespa/slobrok/sbmirror.h +++ b/slobrok/src/vespa/slobrok/sbmirror.h @@ -7,6 +7,7 @@ #include <vespa/vespalib/util/gencnt.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/fnet/frt/invoker.h> +#include <atomic> class FRT_Target; @@ -92,7 +93,7 @@ private: mutable std::mutex _lock; bool _reqPending; bool _scheduled; - bool _reqDone; + std::atomic<bool> _reqDone; bool _logOnSuccess; SpecMap _specs; vespalib::GenCnt _specsGen; diff --git a/slobrok/src/vespa/slobrok/sbregister.cpp b/slobrok/src/vespa/slobrok/sbregister.cpp index cdab4d4009d..c1948a24aa8 100644 --- a/slobrok/src/vespa/slobrok/sbregister.cpp +++ b/slobrok/src/vespa/slobrok/sbregister.cpp @@ -125,8 +125,8 @@ RegisterAPI::unregisterName(vespalib::stringref name) void RegisterAPI::handleReqDone() { - if (_reqDone) { - _reqDone = false; + if (_reqDone.load(std::memory_order_relaxed)) { + _reqDone.store(false, std::memory_order_relaxed); if (_req->IsError()) { if (_req->GetErrorCode() != FRTE_RPC_METHOD_FAILED) { LOG(debug, "register failed: %s (code %d)", @@ -270,9 +270,9 @@ RegisterAPI::PerformTask() void RegisterAPI::RequestDone(FRT_RPCRequest *req) { - LOG_ASSERT(req == _req && !_reqDone); + LOG_ASSERT(req == _req && !_reqDone.load(std::memory_order_relaxed)); (void) req; - _reqDone = true; + _reqDone.store(true, std::memory_order_relaxed); ScheduleNow(); } diff --git a/slobrok/src/vespa/slobrok/sbregister.h b/slobrok/src/vespa/slobrok/sbregister.h index 7590cbc0afa..16f14446234 100644 --- a/slobrok/src/vespa/slobrok/sbregister.h +++ b/slobrok/src/vespa/slobrok/sbregister.h @@ -86,7 +86,7 @@ private: FRT_Supervisor &_orb; RPCHooks _hooks; std::mutex _lock; - bool _reqDone; + std::atomic<bool> _reqDone; bool _logOnSuccess; std::atomic<bool> _busy; SlobrokList _slobrokSpecs; diff --git a/slobrok/src/vespa/slobrok/server/metrics_producer.h b/slobrok/src/vespa/slobrok/server/metrics_producer.h index aade6f25c76..c25f3653dc3 100644 --- a/slobrok/src/vespa/slobrok/server/metrics_producer.h +++ b/slobrok/src/vespa/slobrok/server/metrics_producer.h @@ -2,8 +2,8 @@ #pragma once #include "rpchooks.h" -#include <vespa/vespalib/net/metrics_producer.h> -#include <vespa/vespalib/net/simple_metrics_producer.h> +#include <vespa/vespalib/net/http/metrics_producer.h> +#include <vespa/vespalib/net/http/simple_metrics_producer.h> class FNET_Transport; diff --git a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp index 1bf6596744d..2c64c7fa2df 100644 --- a/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp +++ b/slobrok/src/vespa/slobrok/server/reconfigurable_stateserver.cpp @@ -2,7 +2,7 @@ #include "reconfigurable_stateserver.h" #include <vespa/vespalib/util/exceptions.h> -#include <vespa/vespalib/net/state_server.h> +#include <vespa/vespalib/net/http/state_server.h> #include <vespa/config/helper/configfetcher.hpp> #include <thread> diff --git a/slobrok/src/vespa/slobrok/server/sbenv.h b/slobrok/src/vespa/slobrok/server/sbenv.h index aac2d3a0d92..653d4b75818 100644 --- a/slobrok/src/vespa/slobrok/server/sbenv.h +++ b/slobrok/src/vespa/slobrok/server/sbenv.h @@ -12,8 +12,8 @@ #include "union_service_map.h" #include <vespa/config-slobroks.h> #include <vespa/slobrok/cfg.h> -#include <vespa/vespalib/net/simple_health_producer.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/net/http/simple_health_producer.h> +#include <vespa/vespalib/net/http/simple_component_config_producer.h> class FastOS_ThreadPool; class FNET_Transport; diff --git a/staging_vespalib/.gitignore b/staging_vespalib/.gitignore deleted file mode 100644 index 65e000de4f0..00000000000 --- a/staging_vespalib/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -bin -doc -lib -Makefile diff --git a/staging_vespalib/CMakeLists.txt b/staging_vespalib/CMakeLists.txt deleted file mode 100644 index ff152bcc2a1..00000000000 --- a/staging_vespalib/CMakeLists.txt +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") -set(STAGING_VESPALIB_DIRECTIO_TESTDIR src/tests/directio) -set(STAGING_VESPALIB_PROCESS_MEMORY_STATS_TESTDIR src/tests/util/process_memory_stats) -endif() - -vespa_define_module( - DEPENDS - fastos - vespalog - vespalib - fnet - - TESTS - src/tests/array - src/tests/benchmark - src/tests/bits - src/tests/clock - src/tests/crc - ${STAGING_VESPALIB_DIRECTIO_TESTDIR} - src/tests/encoding/base64 - src/tests/fileheader - src/tests/floatingpointtype - src/tests/growablebytebuffer - src/tests/json - src/tests/memorydatastore - src/tests/metrics - src/tests/objectdump - src/tests/objects - src/tests/objectselection - src/tests/polymorphicarray - src/tests/programoptions - src/tests/rusage - src/tests/shutdownguard - src/tests/state_server - src/tests/stllike - src/tests/sequencedtaskexecutor - src/tests/singleexecutor - src/tests/timer - ${STAGING_VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} - src/tests/xmlserializable - - LIBS - src/vespa/vespalib - src/vespa/vespalib/data - src/vespa/vespalib/encoding - src/vespa/vespalib/metrics - src/vespa/vespalib/net - src/vespa/vespalib/objects - src/vespa/vespalib/stllike - src/vespa/vespalib/util -) diff --git a/staging_vespalib/OWNERS b/staging_vespalib/OWNERS deleted file mode 100644 index 3853ae0ffae..00000000000 --- a/staging_vespalib/OWNERS +++ /dev/null @@ -1,3 +0,0 @@ -arnej27959 -havardpe -baldersheim diff --git a/staging_vespalib/README b/staging_vespalib/README deleted file mode 100644 index 4cb562d8b79..00000000000 --- a/staging_vespalib/README +++ /dev/null @@ -1,3 +0,0 @@ -Vespalib is the location for common and generic vespa utility classes. -This is the staging location for classes that are intended to be added -to vespalib, but aren't finished enough yet.
\ No newline at end of file diff --git a/staging_vespalib/src/.gitignore b/staging_vespalib/src/.gitignore deleted file mode 100644 index 7871665637b..00000000000 --- a/staging_vespalib/src/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -*.dsp -Makefile.ini -config_command.sh -project.dsw -/module.conf diff --git a/staging_vespalib/src/Doxyfile b/staging_vespalib/src/Doxyfile deleted file mode 100644 index 6ad88d3127b..00000000000 --- a/staging_vespalib/src/Doxyfile +++ /dev/null @@ -1,225 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -# Doxyfile 1.4.6 - -#--------------------------------------------------------------------------- -# Project related configuration options -#--------------------------------------------------------------------------- -PROJECT_NAME = vespalib -PROJECT_NUMBER = -OUTPUT_DIRECTORY = ../doc/doxygen -CREATE_SUBDIRS = NO -OUTPUT_LANGUAGE = English -USE_WINDOWS_ENCODING = NO -BRIEF_MEMBER_DESC = YES -REPEAT_BRIEF = YES -ABBREVIATE_BRIEF = -ALWAYS_DETAILED_SEC = NO -INLINE_INHERITED_MEMB = NO -FULL_PATH_NAMES = YES -STRIP_FROM_PATH = -STRIP_FROM_INC_PATH = ./ -SHORT_NAMES = NO -JAVADOC_AUTOBRIEF = NO -MULTILINE_CPP_IS_BRIEF = NO -DETAILS_AT_TOP = NO -INHERIT_DOCS = YES -SEPARATE_MEMBER_PAGES = NO -TAB_SIZE = 8 -ALIASES = -OPTIMIZE_OUTPUT_FOR_C = NO -OPTIMIZE_OUTPUT_JAVA = NO -BUILTIN_STL_SUPPORT = NO -DISTRIBUTE_GROUP_DOC = NO -SUBGROUPING = YES -#--------------------------------------------------------------------------- -# Build related configuration options -#--------------------------------------------------------------------------- -EXTRACT_ALL = YES -EXTRACT_PRIVATE = NO -EXTRACT_STATIC = NO -EXTRACT_LOCAL_CLASSES = YES -EXTRACT_LOCAL_METHODS = NO -HIDE_UNDOC_MEMBERS = YES -HIDE_UNDOC_CLASSES = YES -HIDE_FRIEND_COMPOUNDS = NO -HIDE_IN_BODY_DOCS = NO -INTERNAL_DOCS = NO -CASE_SENSE_NAMES = YES -HIDE_SCOPE_NAMES = NO -SHOW_INCLUDE_FILES = YES -INLINE_INFO = YES -SORT_MEMBER_DOCS = YES -SORT_BRIEF_DOCS = NO -SORT_BY_SCOPE_NAME = NO -GENERATE_TODOLIST = YES -GENERATE_TESTLIST = YES -GENERATE_BUGLIST = YES -GENERATE_DEPRECATEDLIST= YES -ENABLED_SECTIONS = -MAX_INITIALIZER_LINES = 30 -SHOW_USED_FILES = YES -SHOW_DIRECTORIES = NO -FILE_VERSION_FILTER = -#--------------------------------------------------------------------------- -# configuration options related to warning and progress messages -#--------------------------------------------------------------------------- -QUIET = NO -WARNINGS = YES -WARN_IF_UNDOCUMENTED = YES -WARN_IF_DOC_ERROR = YES -WARN_NO_PARAMDOC = NO -WARN_FORMAT = "$file:$line: $text" -WARN_LOGFILE = -#--------------------------------------------------------------------------- -# configuration options related to the input files -#--------------------------------------------------------------------------- -INPUT = vespalib/util \ - vespalib/testkit -FILE_PATTERNS = -RECURSIVE = NO -EXCLUDE = -EXCLUDE_SYMLINKS = NO -EXCLUDE_PATTERNS = -EXAMPLE_PATH = -EXAMPLE_PATTERNS = -EXAMPLE_RECURSIVE = NO -IMAGE_PATH = -INPUT_FILTER = -FILTER_PATTERNS = -FILTER_SOURCE_FILES = NO -#--------------------------------------------------------------------------- -# configuration options related to source browsing -#--------------------------------------------------------------------------- -SOURCE_BROWSER = YES -INLINE_SOURCES = NO -STRIP_CODE_COMMENTS = YES -REFERENCED_BY_RELATION = NO -REFERENCES_RELATION = NO -USE_HTAGS = NO -VERBATIM_HEADERS = YES -#--------------------------------------------------------------------------- -# configuration options related to the alphabetical class index -#--------------------------------------------------------------------------- -ALPHABETICAL_INDEX = YES -COLS_IN_ALPHA_INDEX = 5 -IGNORE_PREFIX = -#--------------------------------------------------------------------------- -# configuration options related to the HTML output -#--------------------------------------------------------------------------- -GENERATE_HTML = YES -HTML_OUTPUT = html -HTML_FILE_EXTENSION = .html -HTML_HEADER = -HTML_FOOTER = -HTML_STYLESHEET = -HTML_ALIGN_MEMBERS = YES -GENERATE_HTMLHELP = NO -CHM_FILE = -HHC_LOCATION = -GENERATE_CHI = NO -BINARY_TOC = NO -TOC_EXPAND = NO -DISABLE_INDEX = NO -ENUM_VALUES_PER_LINE = 4 -GENERATE_TREEVIEW = NO -TREEVIEW_WIDTH = 250 -#--------------------------------------------------------------------------- -# configuration options related to the LaTeX output -#--------------------------------------------------------------------------- -GENERATE_LATEX = NO -LATEX_OUTPUT = latex -LATEX_CMD_NAME = latex -MAKEINDEX_CMD_NAME = makeindex -COMPACT_LATEX = NO -PAPER_TYPE = a4wide -EXTRA_PACKAGES = -LATEX_HEADER = -PDF_HYPERLINKS = NO -USE_PDFLATEX = NO -LATEX_BATCHMODE = NO -LATEX_HIDE_INDICES = NO -#--------------------------------------------------------------------------- -# configuration options related to the RTF output -#--------------------------------------------------------------------------- -GENERATE_RTF = NO -RTF_OUTPUT = rtf -COMPACT_RTF = NO -RTF_HYPERLINKS = NO -RTF_STYLESHEET_FILE = -RTF_EXTENSIONS_FILE = -#--------------------------------------------------------------------------- -# configuration options related to the man page output -#--------------------------------------------------------------------------- -GENERATE_MAN = NO -MAN_OUTPUT = man -MAN_EXTENSION = .3 -MAN_LINKS = NO -#--------------------------------------------------------------------------- -# configuration options related to the XML output -#--------------------------------------------------------------------------- -GENERATE_XML = NO -XML_OUTPUT = xml -XML_SCHEMA = -XML_DTD = -XML_PROGRAMLISTING = YES -#--------------------------------------------------------------------------- -# configuration options for the AutoGen Definitions output -#--------------------------------------------------------------------------- -GENERATE_AUTOGEN_DEF = NO -#--------------------------------------------------------------------------- -# configuration options related to the Perl module output -#--------------------------------------------------------------------------- -GENERATE_PERLMOD = NO -PERLMOD_LATEX = NO -PERLMOD_PRETTY = YES -PERLMOD_MAKEVAR_PREFIX = -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- -ENABLE_PREPROCESSING = YES -MACRO_EXPANSION = NO -EXPAND_ONLY_PREDEF = NO -SEARCH_INCLUDES = YES -INCLUDE_PATH = -INCLUDE_FILE_PATTERNS = -PREDEFINED = IAM_DOXYGEN -EXPAND_AS_DEFINED = -SKIP_FUNCTION_MACROS = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to external references -#--------------------------------------------------------------------------- -TAGFILES = -GENERATE_TAGFILE = ../doc/doxygen/vespalib.tag -ALLEXTERNALS = NO -EXTERNAL_GROUPS = YES -PERL_PATH = /usr/bin/perl -#--------------------------------------------------------------------------- -# Configuration options related to the dot tool -#--------------------------------------------------------------------------- -CLASS_DIAGRAMS = YES -HIDE_UNDOC_RELATIONS = YES -HAVE_DOT = YES -CLASS_GRAPH = YES -COLLABORATION_GRAPH = YES -GROUP_GRAPHS = YES -UML_LOOK = NO -TEMPLATE_RELATIONS = NO -INCLUDE_GRAPH = YES -INCLUDED_BY_GRAPH = YES -CALL_GRAPH = NO -GRAPHICAL_HIERARCHY = YES -DIRECTORY_GRAPH = YES -DOT_IMAGE_FORMAT = png -DOT_PATH = -DOTFILE_DIRS = -MAX_DOT_GRAPH_WIDTH = 1024 -MAX_DOT_GRAPH_HEIGHT = 1024 -MAX_DOT_GRAPH_DEPTH = 0 -DOT_TRANSPARENT = NO -DOT_MULTI_TARGETS = NO -GENERATE_LEGEND = YES -DOT_CLEANUP = YES -#--------------------------------------------------------------------------- -# Configuration::additions related to the search engine -#--------------------------------------------------------------------------- -SEARCHENGINE = NO diff --git a/staging_vespalib/src/tests/.gitignore b/staging_vespalib/src/tests/.gitignore deleted file mode 100644 index c473b24b98a..00000000000 --- a/staging_vespalib/src/tests/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -testrunner -*_test diff --git a/staging_vespalib/src/tests/array/.gitignore b/staging_vespalib/src/tests/array/.gitignore deleted file mode 100644 index 154e6464c50..00000000000 --- a/staging_vespalib/src/tests/array/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/sort_benchmark -/allocinarray_benchmark -staging_vespalib_allocinarray_test_app -staging_vespalib_allocinarray_benchmark_app -staging_vespalib_sort_benchmark_app diff --git a/staging_vespalib/src/tests/array/CMakeLists.txt b/staging_vespalib/src/tests/array/CMakeLists.txt deleted file mode 100644 index 74ed249c099..00000000000 --- a/staging_vespalib/src/tests/array/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_allocinarray_test_app TEST - SOURCES - allocinarray_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_allocinarray_test_app COMMAND staging_vespalib_allocinarray_test_app) -vespa_add_executable(staging_vespalib_sort_benchmark_app - SOURCES - sort_benchmark.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_sort_benchmark_app COMMAND staging_vespalib_sort_benchmark_app BENCHMARK) -vespa_add_executable(staging_vespalib_allocinarray_benchmark_app - SOURCES - allocinarray_benchmark.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_allocinarray_benchmark_app COMMAND staging_vespalib_allocinarray_benchmark_app BENCHMARK) diff --git a/staging_vespalib/src/tests/array/allocinarray_benchmark.cpp b/staging_vespalib/src/tests/array/allocinarray_benchmark.cpp deleted file mode 100644 index 801f946687c..00000000000 --- a/staging_vespalib/src/tests/array/allocinarray_benchmark.cpp +++ /dev/null @@ -1,126 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/rusage.h> -#include <vespa/vespalib/util/optimized.h> -#include <vespa/vespalib/util/allocinarray.h> -#include <vespa/vespalib/util/array.hpp> -#include <csignal> - -#include <vespa/log/log.h> -LOG_SETUP("allocinarray_benchmark"); - -using namespace vespalib; - -class Test : public TestApp -{ -public: -private: - void benchmarkTree(size_t count); - void benchmarkTreeInArray(size_t count); - int Main() override; -}; - -template <typename T> -class TreeNode -{ -public: - typedef TreeNode * P; - TreeNode(const T & p) :_left(NULL), _right(NULL), _payLoad(p) { } - ~TreeNode() { - if (_left) { - delete _left; - } - if (_right) { - delete _right; - } - } - P left() { return _left; } - P right() { return _right; } - void left(P l) { _left = l; } - void right(P l) { _right = l; } -private: - P _left; - P _right; - T _payLoad; -}; - -template <typename T> -class RefTreeNode -{ -public: - typedef uint32_t P; - RefTreeNode(const T & p) :_left(-1), _right(-1), _payLoad(p) { } - P left() { return _left; } - P right() { return _right; } - void left(P l) { _left = l; } - void right(P l) { _right = l; } -private: - P _left; - P _right; - T _payLoad; -}; - -typedef TreeNode<long> N; -typedef RefTreeNode<long> R; -typedef AllocInArray<R, vespalib::Array<R> > Store; - -void populate(Store & store, uint32_t parent, size_t depth) -{ - if (depth > 0) { - store[parent].left(store.alloc(R(0))); - populate(store, store[parent].left(), depth-1); - store[parent].right(store.alloc(R(1))); - populate(store, store[parent].right(), depth-1); - } -} - -void populate(N * parent, size_t depth) -{ - if (depth > 0) { - parent->left(new N(0)); - populate(parent->left(), depth-1); - parent->right(new N(1)); - populate(parent->right(), depth-1); - } -} - -void Test::benchmarkTree(size_t count) -{ - N root(0); - size_t depth = Optimized::msbIdx(count); - populate(&root, depth); -} - -void Test::benchmarkTreeInArray(size_t count) -{ - Store store; - store.alloc(R(0)); - size_t depth = Optimized::msbIdx(count); - populate(store, 0, depth); -} - -int -Test::Main() -{ - std::string type("direct"); - size_t count = 1000000; - if (_argc > 1) { - type = _argv[1]; - } - if (_argc > 2) { - count = strtol(_argv[2], NULL, 0); - } - TEST_INIT("allocinarray_benchmark"); - steady_time start(steady_clock::now()); - if (type == "direct") { - benchmarkTree(count); - } else { - benchmarkTreeInArray(count); - } - LOG(info, "rusage = {\n%s\n}", vespalib::RUsage::createSelf(start).toString().c_str()); - ASSERT_EQUAL(0, kill(0, SIGPROF)); - TEST_DONE(); -} - -TEST_APPHOOK(Test); - diff --git a/staging_vespalib/src/tests/array/allocinarray_test.cpp b/staging_vespalib/src/tests/array/allocinarray_test.cpp deleted file mode 100644 index b6acd9e7e78..00000000000 --- a/staging_vespalib/src/tests/array/allocinarray_test.cpp +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/array.h> -#include <vespa/vespalib/util/allocinarray.h> -#include <vespa/vespalib/stllike/string.h> -#include <vespa/vespalib/testkit/testapp.h> -#include <deque> - -using namespace vespalib; - -class Test : public TestApp -{ -public: - int Main() override; -private: - template <typename T, typename V> - void testAllocInArray(); -}; - -int -Test::Main() -{ - TEST_INIT("allocinarray_test"); - - testAllocInArray<int64_t, vespalib::Array<int64_t> >(); - testAllocInArray<int64_t, vespalib::Array<int64_t> >(); - testAllocInArray<int64_t, std::vector<int64_t> >(); - testAllocInArray<int64_t, std::deque<int64_t> >(); - - TEST_DONE(); -} - -template <typename T, typename V> -void Test::testAllocInArray() -{ - typedef AllocInArray<T, V> AA; - AA alloc; - EXPECT_EQUAL(0ul, alloc.size()); - EXPECT_EQUAL(0ul, alloc.alloc(1)); - EXPECT_EQUAL(1ul, alloc.size()); - EXPECT_EQUAL(1, alloc[0]); - alloc.free(0); - EXPECT_EQUAL(0ul, alloc.size()); - alloc.free(0); - EXPECT_EQUAL(0ul, alloc.size()); - alloc.free(1); - EXPECT_EQUAL(0ul, alloc.size()); - - alloc.alloc(7); - alloc.alloc(17); - alloc.alloc(-17); - EXPECT_EQUAL(3ul, alloc.size()); - EXPECT_EQUAL(7, alloc[0]); - EXPECT_EQUAL(17, alloc[1]); - EXPECT_EQUAL(-17, alloc[2]); - alloc[1] = 99; - EXPECT_EQUAL(99, alloc[1]); - alloc.free(1); - EXPECT_EQUAL(2ul, alloc.size()); - EXPECT_EQUAL(7, alloc[0]); - EXPECT_EQUAL(-17, alloc[2]); - EXPECT_EQUAL(1ul, alloc.alloc(103)); - EXPECT_EQUAL(3ul, alloc.size()); - EXPECT_EQUAL(7, alloc[0]); - EXPECT_EQUAL(103, alloc[1]); - EXPECT_EQUAL(-17, alloc[2]); - - alloc.clear(); - EXPECT_EQUAL(0ul, alloc.size()); -} - -TEST_APPHOOK(Test) diff --git a/staging_vespalib/src/tests/benchmark/.gitignore b/staging_vespalib/src/tests/benchmark/.gitignore deleted file mode 100644 index 789e52c1370..00000000000 --- a/staging_vespalib/src/tests/benchmark/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -benchmark_test -staging_vespalib_benchmark_test_app diff --git a/staging_vespalib/src/tests/benchmark/CMakeLists.txt b/staging_vespalib/src/tests/benchmark/CMakeLists.txt deleted file mode 100644 index 95d61495047..00000000000 --- a/staging_vespalib/src/tests/benchmark/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_benchmark_test_app - SOURCES - benchmark.cpp - testbase.cpp - DEPENDS - staging_vespalib - EXTERNAL_DEPENDS - ${VESPA_GLIBC_RT_LIB} -) -vespa_add_test(NAME staging_vespalib_benchmark_test NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_test.sh BENCHMARK - DEPENDS staging_vespalib_benchmark_test_app) diff --git a/staging_vespalib/src/tests/benchmark/benchmark_test.sh b/staging_vespalib/src/tests/benchmark/benchmark_test.sh deleted file mode 100755 index 172a8a1379b..00000000000 --- a/staging_vespalib/src/tests/benchmark/benchmark_test.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -set -e -TIME=time - -$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByReferenceVectorInt 200000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByValueVectorInt 4000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByReferenceVectorString 30000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ParamByValueVectorString 40 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ReturnByReferenceVectorString 10 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ReturnByValueVectorString 10 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ReturnByValueMultiVectorString 10 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockSystem 1000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockGToD 1000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockGToD 20000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockREALTIME 1000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockMONOTONIC 1000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockMONOTONIC_RAW 1000 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockPROCESS_CPUTIME_ID 2500 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::ClockTHREAD_CPUTIME_ID 2500 1 -$TIME ./staging_vespalib_benchmark_test_app vespalib::CreateVespalibString 20000 1 diff --git a/staging_vespalib/src/tests/bits/.gitignore b/staging_vespalib/src/tests/bits/.gitignore deleted file mode 100644 index 82481fe3c30..00000000000 --- a/staging_vespalib/src/tests/bits/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -bits_test -staging_vespalib_bits_test_app diff --git a/staging_vespalib/src/tests/bits/CMakeLists.txt b/staging_vespalib/src/tests/bits/CMakeLists.txt deleted file mode 100644 index 36edcf3bb8f..00000000000 --- a/staging_vespalib/src/tests/bits/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_bits_test_app TEST - SOURCES - bits_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_bits_test_app COMMAND staging_vespalib_bits_test_app) diff --git a/staging_vespalib/src/tests/clock/.gitignore b/staging_vespalib/src/tests/clock/.gitignore deleted file mode 100644 index b0fcd343176..00000000000 --- a/staging_vespalib/src/tests/clock/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.depend -Makefile -clock_test -staging_vespalib_clock_test_app -staging_vespalib_clock_benchmark_app diff --git a/staging_vespalib/src/tests/clock/CMakeLists.txt b/staging_vespalib/src/tests/clock/CMakeLists.txt deleted file mode 100644 index 240ad8d0a55..00000000000 --- a/staging_vespalib/src/tests/clock/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_clock_benchmark_app TEST - SOURCES - clock_benchmark.cpp - DEPENDS - staging_vespalib -) -vespa_add_executable(staging_vespalib_clock_test_app TEST - SOURCES - clock_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_clock_test_app COMMAND staging_vespalib_clock_test_app) diff --git a/staging_vespalib/src/tests/crc/.gitignore b/staging_vespalib/src/tests/crc/.gitignore deleted file mode 100644 index 448fa141b03..00000000000 --- a/staging_vespalib/src/tests/crc/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -crc_test -staging_vespalib_crc_test_app diff --git a/staging_vespalib/src/tests/crc/CMakeLists.txt b/staging_vespalib/src/tests/crc/CMakeLists.txt deleted file mode 100644 index b6c89d00c54..00000000000 --- a/staging_vespalib/src/tests/crc/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_crc_test_app TEST - SOURCES - crc_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_crc_test_app COMMAND staging_vespalib_crc_test_app boost) diff --git a/staging_vespalib/src/tests/directio/.gitignore b/staging_vespalib/src/tests/directio/.gitignore deleted file mode 100644 index 6e6dbe13324..00000000000 --- a/staging_vespalib/src/tests/directio/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_directio_test_app diff --git a/staging_vespalib/src/tests/directio/CMakeLists.txt b/staging_vespalib/src/tests/directio/CMakeLists.txt deleted file mode 100644 index 64da387d950..00000000000 --- a/staging_vespalib/src/tests/directio/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_directio_test_app TEST - SOURCES - directio.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_directio_test_app COMMAND staging_vespalib_directio_test_app) diff --git a/staging_vespalib/src/tests/encoding/base64/.gitignore b/staging_vespalib/src/tests/encoding/base64/.gitignore deleted file mode 100644 index 04f2994a28f..00000000000 --- a/staging_vespalib/src/tests/encoding/base64/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_base64_test_app diff --git a/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt b/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt deleted file mode 100644 index 2c88ab1dc43..00000000000 --- a/staging_vespalib/src/tests/encoding/base64/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_base64_test_app TEST - SOURCES - base64_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_base64_test_app COMMAND staging_vespalib_base64_test_app) diff --git a/staging_vespalib/src/tests/fileheader/CMakeLists.txt b/staging_vespalib/src/tests/fileheader/CMakeLists.txt deleted file mode 100644 index 0312dddb06a..00000000000 --- a/staging_vespalib/src/tests/fileheader/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_fileheader_test_app TEST - SOURCES - fileheader_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_fileheader_test_app COMMAND staging_vespalib_fileheader_test_app) diff --git a/staging_vespalib/src/tests/floatingpointtype/.gitignore b/staging_vespalib/src/tests/floatingpointtype/.gitignore deleted file mode 100644 index f99dd0fd707..00000000000 --- a/staging_vespalib/src/tests/floatingpointtype/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -floatingpointtype_test -staging_vespalib_floatingpointtype_test_app diff --git a/staging_vespalib/src/tests/floatingpointtype/CMakeLists.txt b/staging_vespalib/src/tests/floatingpointtype/CMakeLists.txt deleted file mode 100644 index eca27430e0b..00000000000 --- a/staging_vespalib/src/tests/floatingpointtype/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_floatingpointtype_test_app TEST - SOURCES - floatingpointtypetest.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_floatingpointtype_test_app COMMAND staging_vespalib_floatingpointtype_test_app) diff --git a/staging_vespalib/src/tests/growablebytebuffer/.gitignore b/staging_vespalib/src/tests/growablebytebuffer/.gitignore deleted file mode 100644 index 8df98f682b8..00000000000 --- a/staging_vespalib/src/tests/growablebytebuffer/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -growablebytebuffer_test -staging_vespalib_growablebytebuffer_test_app diff --git a/staging_vespalib/src/tests/growablebytebuffer/CMakeLists.txt b/staging_vespalib/src/tests/growablebytebuffer/CMakeLists.txt deleted file mode 100644 index 9126bd1cd07..00000000000 --- a/staging_vespalib/src/tests/growablebytebuffer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_growablebytebuffer_test_app TEST - SOURCES - growablebytebuffer_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_growablebytebuffer_test_app COMMAND staging_vespalib_growablebytebuffer_test_app) diff --git a/staging_vespalib/src/tests/json/.gitignore b/staging_vespalib/src/tests/json/.gitignore deleted file mode 100644 index 963bafed038..00000000000 --- a/staging_vespalib/src/tests/json/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_json_test_app diff --git a/staging_vespalib/src/tests/json/CMakeLists.txt b/staging_vespalib/src/tests/json/CMakeLists.txt deleted file mode 100644 index 2ca20be9258..00000000000 --- a/staging_vespalib/src/tests/json/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_json_test_app TEST - SOURCES - json.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_json_test_app COMMAND staging_vespalib_json_test_app boost) diff --git a/staging_vespalib/src/tests/memorydatastore/.gitignore b/staging_vespalib/src/tests/memorydatastore/.gitignore deleted file mode 100644 index 634cea1ae88..00000000000 --- a/staging_vespalib/src/tests/memorydatastore/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_memorydatastore_test_app diff --git a/staging_vespalib/src/tests/memorydatastore/CMakeLists.txt b/staging_vespalib/src/tests/memorydatastore/CMakeLists.txt deleted file mode 100644 index f3709aeea65..00000000000 --- a/staging_vespalib/src/tests/memorydatastore/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_memorydatastore_test_app TEST - SOURCES - memorydatastore.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_memorydatastore_test_app COMMAND staging_vespalib_memorydatastore_test_app) diff --git a/staging_vespalib/src/tests/metrics/CMakeLists.txt b/staging_vespalib/src/tests/metrics/CMakeLists.txt deleted file mode 100644 index ae5462c28ea..00000000000 --- a/staging_vespalib/src/tests/metrics/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_metrics_test_app TEST - SOURCES - simple_metrics_test.cpp - mock_tick.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_metrics_test_app COMMAND staging_vespalib_metrics_test_app) - -vespa_add_executable(staging_vespalib_stablestore_test_app TEST - SOURCES - stable_store_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_stablestore_test_app COMMAND staging_vespalib_stablestore_test_app) diff --git a/staging_vespalib/src/tests/objectdump/.gitignore b/staging_vespalib/src/tests/objectdump/.gitignore deleted file mode 100644 index d243567aea6..00000000000 --- a/staging_vespalib/src/tests/objectdump/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -objectdump_test -staging_vespalib_objectdump_test_app diff --git a/staging_vespalib/src/tests/objectdump/CMakeLists.txt b/staging_vespalib/src/tests/objectdump/CMakeLists.txt deleted file mode 100644 index 29fc8a748ee..00000000000 --- a/staging_vespalib/src/tests/objectdump/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_objectdump_test_app TEST - SOURCES - objectdump.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_objectdump_test_app COMMAND staging_vespalib_objectdump_test_app) diff --git a/staging_vespalib/src/tests/objects/CMakeLists.txt b/staging_vespalib/src/tests/objects/CMakeLists.txt deleted file mode 100644 index f1313da4bae..00000000000 --- a/staging_vespalib/src/tests/objects/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_identifiable_test_app TEST - SOURCES - identifiable_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_identifiable_test_app COMMAND staging_vespalib_identifiable_test_app) diff --git a/staging_vespalib/src/tests/objectselection/.gitignore b/staging_vespalib/src/tests/objectselection/.gitignore deleted file mode 100644 index 3dece3fbbfe..00000000000 --- a/staging_vespalib/src/tests/objectselection/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -objectselection_test -staging_vespalib_objectselection_test_app diff --git a/staging_vespalib/src/tests/objectselection/CMakeLists.txt b/staging_vespalib/src/tests/objectselection/CMakeLists.txt deleted file mode 100644 index 63f33d05e62..00000000000 --- a/staging_vespalib/src/tests/objectselection/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_objectselection_test_app TEST - SOURCES - objectselection.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_objectselection_test_app COMMAND staging_vespalib_objectselection_test_app) diff --git a/staging_vespalib/src/tests/polymorphicarray/.gitignore b/staging_vespalib/src/tests/polymorphicarray/.gitignore deleted file mode 100644 index b3a74390312..00000000000 --- a/staging_vespalib/src/tests/polymorphicarray/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_polymorphicarray_test_app diff --git a/staging_vespalib/src/tests/polymorphicarray/CMakeLists.txt b/staging_vespalib/src/tests/polymorphicarray/CMakeLists.txt deleted file mode 100644 index 3c7e6074b88..00000000000 --- a/staging_vespalib/src/tests/polymorphicarray/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_polymorphicarray_test_app TEST - SOURCES - polymorphicarray_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_polymorphicarray_test_app COMMAND staging_vespalib_polymorphicarray_test_app) diff --git a/staging_vespalib/src/tests/programoptions/.gitignore b/staging_vespalib/src/tests/programoptions/.gitignore deleted file mode 100644 index 31f3aa61556..00000000000 --- a/staging_vespalib/src/tests/programoptions/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.depend -Makefile -programoptions_test -staging_vespalib_programoptions_test_app diff --git a/staging_vespalib/src/tests/programoptions/CMakeLists.txt b/staging_vespalib/src/tests/programoptions/CMakeLists.txt deleted file mode 100644 index a4d71815621..00000000000 --- a/staging_vespalib/src/tests/programoptions/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_programoptions_test_app TEST - SOURCES - programoptions_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_programoptions_test_app COMMAND staging_vespalib_programoptions_test_app) diff --git a/staging_vespalib/src/tests/rusage/.gitignore b/staging_vespalib/src/tests/rusage/.gitignore deleted file mode 100644 index 195922a9ced..00000000000 --- a/staging_vespalib/src/tests/rusage/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_rusage_test_app diff --git a/staging_vespalib/src/tests/rusage/CMakeLists.txt b/staging_vespalib/src/tests/rusage/CMakeLists.txt deleted file mode 100644 index b52a47183b0..00000000000 --- a/staging_vespalib/src/tests/rusage/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_rusage_test_app TEST - SOURCES - rusage_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_rusage_test_app COMMAND staging_vespalib_rusage_test_app) diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore b/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore deleted file mode 100644 index 523cfe5e3e1..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -staging_vespalib_sequencedtaskexecutor_test_app -staging_vespalib_sequencedtaskexecutor_benchmark_app -staging_vespalib_adaptive_sequenced_executor_test_app -staging_vespalib_foregroundtaskexecutor_test_app diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt b/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt deleted file mode 100644 index cc851f668da..00000000000 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_sequencedtaskexecutor_benchmark_app TEST - SOURCES - sequencedtaskexecutor_benchmark.cpp - DEPENDS - staging_vespalib -) - -vespa_add_executable(staging_vespalib_sequencedtaskexecutor_test_app TEST - SOURCES - sequencedtaskexecutor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_sequencedtaskexecutor_test_app COMMAND staging_vespalib_sequencedtaskexecutor_test_app) - -vespa_add_executable(staging_vespalib_adaptive_sequenced_executor_test_app TEST - SOURCES - adaptive_sequenced_executor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_adaptive_sequenced_executor_test_app COMMAND staging_vespalib_adaptive_sequenced_executor_test_app) - -vespa_add_executable(staging_vespalib_foregroundtaskexecutor_test_app TEST - SOURCES - foregroundtaskexecutor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_foregroundtaskexecutor_test_app COMMAND staging_vespalib_foregroundtaskexecutor_test_app) diff --git a/staging_vespalib/src/tests/shutdownguard/.gitignore b/staging_vespalib/src/tests/shutdownguard/.gitignore deleted file mode 100644 index a596164ac7b..00000000000 --- a/staging_vespalib/src/tests/shutdownguard/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_shutdownguard_test_app diff --git a/staging_vespalib/src/tests/shutdownguard/CMakeLists.txt b/staging_vespalib/src/tests/shutdownguard/CMakeLists.txt deleted file mode 100644 index b1d43788957..00000000000 --- a/staging_vespalib/src/tests/shutdownguard/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_shutdownguard_test_app TEST - SOURCES - shutdownguard_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_shutdownguard_test_app NO_VALGRIND COMMAND staging_vespalib_shutdownguard_test_app) diff --git a/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt b/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt deleted file mode 100644 index 8ff0ff1989a..00000000000 --- a/staging_vespalib/src/tests/singleexecutor/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_singleexecutor_test_app TEST - SOURCES - singleexecutor_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_singleexecutor_test_app COMMAND staging_vespalib_singleexecutor_test_app) diff --git a/staging_vespalib/src/tests/state_server/.gitignore b/staging_vespalib/src/tests/state_server/.gitignore deleted file mode 100644 index b1884d4e060..00000000000 --- a/staging_vespalib/src/tests/state_server/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_state_server_test_app diff --git a/staging_vespalib/src/tests/state_server/CMakeLists.txt b/staging_vespalib/src/tests/state_server/CMakeLists.txt deleted file mode 100644 index fc22d8b9178..00000000000 --- a/staging_vespalib/src/tests/state_server/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_state_server_test_app TEST - SOURCES - state_server_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_state_server_test_app NO_VALGRIND NO_VALGRIND COMMAND staging_vespalib_state_server_test_app ENVIRONMENT "VESPA_HOME=.") diff --git a/staging_vespalib/src/tests/stllike/.gitignore b/staging_vespalib/src/tests/stllike/.gitignore deleted file mode 100644 index 268b9e88393..00000000000 --- a/staging_vespalib/src/tests/stllike/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -.depend -Makefile -lrucache_test -cache_test -staging_vespalib_cache_test_app -staging_vespalib_lrucache_test_app diff --git a/staging_vespalib/src/tests/stllike/CMakeLists.txt b/staging_vespalib/src/tests/stllike/CMakeLists.txt deleted file mode 100644 index 40dc7a8a438..00000000000 --- a/staging_vespalib/src/tests/stllike/CMakeLists.txt +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_lrucache_test_app TEST - SOURCES - lrucache.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_lrucache_test_app COMMAND staging_vespalib_lrucache_test_app) -vespa_add_executable(staging_vespalib_cache_test_app TEST - SOURCES - cache_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_cache_test_app COMMAND staging_vespalib_cache_test_app) diff --git a/staging_vespalib/src/tests/timer/.gitignore b/staging_vespalib/src/tests/timer/.gitignore deleted file mode 100644 index 9031e40152a..00000000000 --- a/staging_vespalib/src/tests/timer/.gitignore +++ /dev/null @@ -1 +0,0 @@ -staging_vespalib_timer_test_app diff --git a/staging_vespalib/src/tests/timer/CMakeLists.txt b/staging_vespalib/src/tests/timer/CMakeLists.txt deleted file mode 100644 index 9fd12f11c7c..00000000000 --- a/staging_vespalib/src/tests/timer/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_timer_test_app TEST - SOURCES - timer_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_timer_test_app COMMAND staging_vespalib_timer_test_app) diff --git a/staging_vespalib/src/tests/util/process_memory_stats/.gitignore b/staging_vespalib/src/tests/util/process_memory_stats/.gitignore deleted file mode 100644 index e32e9f22dd3..00000000000 --- a/staging_vespalib/src/tests/util/process_memory_stats/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -mapfile -staging_vespalib_process_memory_stats_test_app diff --git a/staging_vespalib/src/tests/util/process_memory_stats/CMakeLists.txt b/staging_vespalib/src/tests/util/process_memory_stats/CMakeLists.txt deleted file mode 100644 index 6709c7bac1b..00000000000 --- a/staging_vespalib/src/tests/util/process_memory_stats/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_process_memory_stats_test_app TEST - SOURCES - process_memory_stats_test.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_process_memory_stats_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/process_memory_stats_test.sh - DEPENDS staging_vespalib_process_memory_stats_test_app) diff --git a/staging_vespalib/src/tests/xmlserializable/.gitignore b/staging_vespalib/src/tests/xmlserializable/.gitignore deleted file mode 100644 index d2910cb7407..00000000000 --- a/staging_vespalib/src/tests/xmlserializable/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*_test -.depend -Makefile -staging_vespalib_xmlserializable_test_app diff --git a/staging_vespalib/src/tests/xmlserializable/CMakeLists.txt b/staging_vespalib/src/tests/xmlserializable/CMakeLists.txt deleted file mode 100644 index 999bc45ad66..00000000000 --- a/staging_vespalib/src/tests/xmlserializable/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_executable(staging_vespalib_xmlserializable_test_app TEST - SOURCES - xmlserializabletest.cpp - DEPENDS - staging_vespalib -) -vespa_add_test(NAME staging_vespalib_xmlserializable_test_app COMMAND staging_vespalib_xmlserializable_test_app) diff --git a/staging_vespalib/src/vespa/vespalib/.gitignore b/staging_vespalib/src/vespa/vespalib/.gitignore deleted file mode 100644 index fae8ae72fc9..00000000000 --- a/staging_vespalib/src/vespa/vespalib/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.so.* -.depend -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/CMakeLists.txt deleted file mode 100644 index 43492d06689..00000000000 --- a/staging_vespalib/src/vespa/vespalib/CMakeLists.txt +++ /dev/null @@ -1,13 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib - SOURCES - $<TARGET_OBJECTS:staging_vespalib_vespalib_encoding> - $<TARGET_OBJECTS:staging_vespalib_vespalib_util> - $<TARGET_OBJECTS:staging_vespalib_vespalib_data> - $<TARGET_OBJECTS:staging_vespalib_vespalib_metrics> - $<TARGET_OBJECTS:staging_vespalib_vespalib_objects> - $<TARGET_OBJECTS:staging_vespalib_vespalib_stllike> - $<TARGET_OBJECTS:staging_vespalib_vespalib_net> - INSTALL lib64 - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/data/.gitignore b/staging_vespalib/src/vespa/vespalib/data/.gitignore deleted file mode 100644 index 0b3af54ee50..00000000000 --- a/staging_vespalib/src/vespa/vespalib/data/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.So -.depend* -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt deleted file mode 100644 index d5adeadfb3b..00000000000 --- a/staging_vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_data OBJECT - SOURCES - fileheader.cpp - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/objects/.gitignore b/staging_vespalib/src/vespa/vespalib/objects/.gitignore deleted file mode 100644 index ee8938b6bf4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/objects/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.So -*.exe -*.ilk -*.pdb -.depend* -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/objects/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/objects/CMakeLists.txt deleted file mode 100644 index 6300cad0c72..00000000000 --- a/staging_vespalib/src/vespa/vespalib/objects/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_objects OBJECT - SOURCES - identifiable.cpp - namedobject.cpp - objectvisitor.cpp - objectdumper.cpp - object2slime.cpp - visit.cpp - objectpredicate.cpp - objectoperation.cpp - nboserializer.cpp - serializer.cpp - deserializer.cpp - asciiserializer.cpp - floatingpointtype.cpp - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/objects/memberinfo.h b/staging_vespalib/src/vespa/vespalib/objects/memberinfo.h deleted file mode 100644 index 4f2736cdbf3..00000000000 --- a/staging_vespalib/src/vespa/vespalib/objects/memberinfo.h +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -namespace vespalib { - -#define MI_TYPEID(t) MemberInfo::##t##_id -#define MI_M_OFFSET(c, m) static_cast<long>(&static_cast<const c *>(0)->m) -#define MI_M_SIZEOF(c, m) sizeof(static_cast<const c *>(0)->m) -#define MI_BASE_OFFSET(c,b) static_cast<long>(static_cast<const b *>(static_cast<const * c>(1)) - 1) - -#define MI_DECLARE \ - static const MemberInfo _memberInfo[]; \ - static const unsigned _memberCount; - -#define MI_IMPL_BEGIN(c) const MemberInfo c::_memberInfo[] = { \ -#define MI_IMPL_MEM(c, t, m) {#m, MI_TYPEID(t), MI_M_OFFSET(c, m), sizeof(m) } -#define MI_IMPL_END(c) }; const unsigned c::_memberCount = sizeof(cl::__memberDescription)/sizeof(cl::__memberDescription[0]) - - -class MemberInfo { - public: - enum TypeId { - bool_id = 1, - char_id, - int8_t_id, - uint8_t_id, - int16_t_id, - uint16_t_id, - int32_t_id, - uint32_t_id, - int64_t_id, - uint64_t_id, - float_id, - double_id, - std_string_id, - vespalib_Identifiable_id - }; - const char * _name; - TypeId _type; - size_t _offset - size_t _sizeof; -}; - -} - diff --git a/staging_vespalib/src/vespa/vespalib/objects/simpledynamicdata.h b/staging_vespalib/src/vespa/vespalib/objects/simpledynamicdata.h deleted file mode 100644 index f07832465e7..00000000000 --- a/staging_vespalib/src/vespa/vespalib/objects/simpledynamicdata.h +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vector> -#include <algorithm> -#include <memory> -#include "dynamicdatavalue.h" - -namespace vespalib -{ - -class SimpleDynamicData : public DynamicDataValue -{ -private: - struct DataUnit { - uint32_t id; - Value *v; - DataUnit(uint32_t id_, Value *v_) : id(id_), v(v_) {} - }; - typedef std::vector<DataUnit> DataUnits; - DataUnits _values; - - virtual bool setValueIfExisting(uint32_t id, const Value& v) { - for (DataUnits::iterator it = _values.begin(); it != _values.end(); ++it) { - if (it->id == id) { - delete it->v; - it->v = v.clone(); - return true; - } - } - return false; - } - virtual void addNewValue(uint32_t id, const Value& v) { - _values.push_back(DataUnit(id, v.clone())); - } - - void swap(SimpleDynamicData& other) { - std::swap(_values, other._values); - } - -public: - virtual ~SimpleDynamicData() { - // we own contained data, must delete it - for (DataUnits::iterator it = _values.begin(); it != _values.end(); ++it) { - delete it->v; - } - } - virtual bool hasValue(uint32_t id) const { - for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { - if (it->id == id) return true; - } - return false; - } - virtual void deleteValue(uint32_t id) { - for (DataUnits::iterator it = _values.begin(); it != _values.end(); ++it) { - if (it->id == id) { - delete it->v; - _values.erase(it); - return; - } - } - } - virtual const Value& getValue(uint32_t id) const { - for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { - if (it->id == id) { - return *(it->v); - } - } - throw IllegalArgumentException("id not found"); - } - virtual Value *getValueRef(uint32_t id) const { - for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { - if (it->id == id) { - return it->v; - } - } - return NULL; - } - virtual void visitValues(ValueReceiverI& visitor) const { - for (DataUnits::const_iterator it = _values.begin(); it != _values.end(); ++it) { - visitor(it->id, *(it->v)); - } - } - - SimpleDynamicData() : _values() {} - SimpleDynamicData(const SimpleDynamicData& other) : - DynamicDataValue(), - _values() - { - for (DataUnits::const_iterator it = other._values.begin(); - it != other._values.end(); - ++it) - { - addNewValue(it->id, *(it->v)); - } - } - - SimpleDynamicData& operator= (const SimpleDynamicData& other) { - SimpleDynamicData tmp(other); - swap(tmp); - return *this; - } - - virtual SimpleDynamicData *clone() const { - SimpleDynamicData *ret = new SimpleDynamicData(*this); - return ret; - } -}; - - - -} // namespace vespalib - diff --git a/staging_vespalib/src/vespa/vespalib/objects/valuetype.cpp b/staging_vespalib/src/vespa/vespalib/objects/valuetype.cpp deleted file mode 100644 index 96cf3fa22e1..00000000000 --- a/staging_vespalib/src/vespa/vespalib/objects/valuetype.cpp +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include "valuetype.h" - -namespace vespalib { - -const StringValueType StringValueType::t; -const DynamicDataValueType DynamicDataValueType::t; -const Int16ValueType Int16ValueType::t; -const Int32ValueType Int32ValueType::t; -const Int64ValueType Int64ValueType::t; -const UnsignedByteValueType UnsignedByteValueType::t; -const Uint128ValueType Uint128ValueType::t; -const Uint256ValueType Uint256ValueType::t; -const Uint96ValueType Uint96ValueType::t; -const ArrayOfStringValueType ArrayOfStringValueType::t; -const ArrayOfDynamicDataValueType ArrayOfDynamicDataValueType::t; -const ArrayOfInt16ValueType ArrayOfInt16ValueType::t; -const ArrayOfInt32ValueType ArrayOfInt32ValueType::t; -const ArrayOfInt64ValueType ArrayOfInt64ValueType::t; -const ArrayOfUnsignedByteValueType ArrayOfUnsignedByteValueType::t; -const ArrayOfUint128ValueType ArrayOfUint128ValueType::t; -const ArrayOfUint256ValueType ArrayOfUint256ValueType::t; -const ArrayOfUint96ValueType ArrayOfUint96ValueType::t; - -} // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/stllike/.gitignore b/staging_vespalib/src/vespa/vespalib/stllike/.gitignore deleted file mode 100644 index ee8938b6bf4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/stllike/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.So -*.exe -*.ilk -*.pdb -.depend* -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/stllike/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/stllike/CMakeLists.txt deleted file mode 100644 index a76f2b82d4e..00000000000 --- a/staging_vespalib/src/vespa/vespalib/stllike/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_stllike OBJECT - SOURCES - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/util/.gitignore b/staging_vespalib/src/vespa/vespalib/util/.gitignore deleted file mode 100644 index ee8938b6bf4..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -*.So -*.exe -*.ilk -*.pdb -.depend* -Makefile diff --git a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt b/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt deleted file mode 100644 index 2912ac2397b..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_util OBJECT - SOURCES - adaptive_sequenced_executor.cpp - bits.cpp - clock.cpp - crc.cpp - document_runnable.cpp - doom.cpp - foregroundtaskexecutor.cpp - growablebytebuffer.cpp - isequencedtaskexecutor.cpp - jsonexception.cpp - jsonstream.cpp - jsonwriter.cpp - malloc_mmap_guard.cpp - process_memory_stats.cpp - programoptions.cpp - programoptions_testutils.cpp - rusage.cpp - sequencedtaskexecutor.cpp - sequencedtaskexecutorobserver.cpp - shutdownguard.cpp - scheduledexecutor.cpp - singleexecutor.cpp - testclock.cpp - xmlserializable.cpp - xmlstream.cpp - DEPENDS -) diff --git a/staging_vespalib/src/vespa/vespalib/util/allocinarray.h b/staging_vespalib/src/vespa/vespalib/util/allocinarray.h deleted file mode 100644 index aa8951fe9ec..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/allocinarray.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#pragma once - -#include <vespa/vespalib/util/alloc.h> -#include <vespa/vespalib/util/array.h> - -namespace vespalib { - -/** - * This is a class that lets you allocate memory tightly. It uses an vector interface - * for storing objects of the same type efficiently. It also providex vector like access with []. - * New objects are just appended to the backing vector, or if there are holes they are inserted there. - * freed objects are not destructed upon free, but rather when the place is required again. - * That happens either - * - on free if it is the last element in the backing vector. - * - when the AllocInArray goes out of scope. - * - on an explicit clear. -**/ -template <typename T, typename V=vespalib::Array<T> > -class AllocInArray { -public: - typedef uint32_t Index; - void reserve(size_t sz) { _v.reserve(sz); } - Index alloc(const T & v); - void free(Index p); - const T & operator [] (Index p) const { return _v[p]; } - T & operator [] (Index p) { return _v[p]; } - void clear(); - size_t size() const { return _v.size() - _free.size(); } -private: - Index last() const { return _v.size() - 1; } - typedef vespalib::Array<Index> FreeList; - V _v; - FreeList _free; -}; - -template <typename T, typename V> -typename AllocInArray<T, V>::Index -AllocInArray<T, V>::alloc(const T & v) -{ - if (_free.empty()) { - _v.push_back(v); - return last(); - } else { - Index p(_free.back()); - _free.pop_back(); - _v[p] = v; - return p; - } -} - -template <typename T, typename V> -void -AllocInArray<T, V>::free(Index p) -{ - if (p == last()) { - _v.pop_back(); - } else if ( p < _v.size()) { - _free.push_back(p); - } -} - -template <typename T, typename V> -void -AllocInArray<T, V>::clear() -{ - _v.clear(); - _free.clear(); -} - -} - diff --git a/staging_vespalib/src/vespa/vespalib/util/overview.h b/staging_vespalib/src/vespa/vespalib/util/overview.h deleted file mode 100644 index b998356ba50..00000000000 --- a/staging_vespalib/src/vespa/vespalib/util/overview.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -/*! \mainpage Vespalib - C++ utility library for Vespa components - * - * \section intro_sec Introduction - * - * vespalib is a collection of simple utility classes shared - * between most Vespa components that are written in C++. - * Most of these aren't Vespa specific in any way, except that - * they reflect the Vespa code standard and conventions. - * - * \section install_sec Installation - * - * install vespa_vespalib_dev - * - * \section overview_sec Overview - * - * Most of the classes in vespalib can be used by themselves, - * for a full list see the alphabetical class list. - * Here are the major groups of classes: - * - * Generation counter - * - * vespalib::GenCnt - * - * Reference counting and atomic operations - * - * <BR> vespalib::ReferenceCounter - * - * Simple hashmap - * - * \ref vespalib::HashMap<T> - * - * Scope guards for easier exception-safety - * - * vespalib::CounterGuard - * <BR> vespalib::DirPointer - * <BR> vespalib::FileDescriptor - * <BR> vespalib::FilePointer - * <BR> \ref vespalib::MaxValueGuard<T> - * <BR> \ref vespalib::ValueGuard<T> - * - * General utility macros - * <BR> \ref VESPA_STRLOC - * <BR> \ref VESPA_STRINGIZE(str) - * - * Standalone testing framework - * - * vespalib::TestApp - */ diff --git a/staging_vespalib/testrun/.gitignore b/staging_vespalib/testrun/.gitignore deleted file mode 100644 index faed45bc94a..00000000000 --- a/staging_vespalib/testrun/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -test-report.html -test-report.html.* -test.*.*.desc -test.*.*.file.* -test.*.*.files.html -test.*.*.log -tmp.* -xsync.log -/test.*.*.result -Makefile diff --git a/standalone-container/CMakeLists.txt b/standalone-container/CMakeLists.txt index c3026df11a4..9dbe9aa41db 100644 --- a/standalone-container/CMakeLists.txt +++ b/standalone-container/CMakeLists.txt @@ -1,3 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(standalone-container) +install_jar(standalone-container-jar-with-dependencies.jar) install(PROGRAMS src/main/sh/standalone-container.sh DESTINATION libexec/vespa) diff --git a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java index fe3e8782973..a677c69cb79 100644 --- a/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java +++ b/standalone-container/src/main/java/com/yahoo/container/standalone/StandaloneContainerApplication.java @@ -154,6 +154,7 @@ public class StandaloneContainerApplication implements Application { public void start() { try { com.yahoo.container.Container.get().setCustomFileAcquirer(distributedFiles); + com.yahoo.container.Container.get().disableUrlDownloader(); configuredApplication.start(); } catch (Exception e) { com.yahoo.container.Container.resetInstance(); diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt index e45c78f68b9..4b7c12b0f31 100644 --- a/storage/CMakeLists.txt +++ b/storage/CMakeLists.txt @@ -2,9 +2,7 @@ vespa_define_module( DEPENDS vespadefaults - storageapi fastos - fastlib_fast metrics config_cloudconfig configdefinitions @@ -15,7 +13,6 @@ vespa_define_module( vespalib vdslib persistence - storageframework EXTERNAL_DEPENDS ${VESPA_GLIBC_RT_LIB} @@ -42,6 +39,19 @@ vespa_define_module( src/vespa/storage/storageutil src/vespa/storage/tools src/vespa/storage/visiting + src/vespa/storageframework/defaultimplementation/clock + src/vespa/storageframework/defaultimplementation/component + src/vespa/storageframework/defaultimplementation/thread + src/vespa/storageframework/generic/clock + src/vespa/storageframework/generic/component + src/vespa/storageframework/generic/metric + src/vespa/storageframework/generic/status + src/vespa/storageframework/generic/thread + src/vespa/storageapi/app + src/vespa/storageapi/buckets + src/vespa/storageapi/mbusprot + src/vespa/storageapi/message + src/vespa/storageapi/messageapi TEST_DEPENDS messagebus_messagebus-test @@ -59,6 +69,13 @@ vespa_define_module( src/tests/persistence src/tests/persistence/common src/tests/persistence/filestorage + src/tests/storageapi + src/tests/storageapi/buckets + src/tests/storageapi/mbusprot + src/tests/storageapi/messageapi + src/tests/storageframework + src/tests/storageframework/clock + src/tests/storageframework/thread src/tests/storageserver src/tests/storageserver/rpc src/tests/visiting diff --git a/storage/src/tests/common/teststorageapp.h b/storage/src/tests/common/teststorageapp.h index 04cce2470f4..de1dc99bb6e 100644 --- a/storage/src/tests/common/teststorageapp.h +++ b/storage/src/tests/common/teststorageapp.h @@ -20,7 +20,7 @@ #include "testnodestateupdater.h" #include <vespa/document/base/testdocman.h> #include <vespa/document/bucket/fixed_bucket_spaces.h> -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> #include <vespa/storage/bucketdb/storbucketdb.h> #include <vespa/storage/common/doneinitializehandler.h> #include <vespa/storage/common/hostreporter/hostinfo.h> diff --git a/storage/src/tests/distributor/CMakeLists.txt b/storage/src/tests/distributor/CMakeLists.txt index 501f9a18c47..c8dde75c73d 100644 --- a/storage/src/tests/distributor/CMakeLists.txt +++ b/storage/src/tests/distributor/CMakeLists.txt @@ -55,7 +55,7 @@ vespa_add_executable(storage_distributor_gtest_runner_app TEST DEPENDS storage_testcommon storage_testhostreporter - storage_distributor + storage GTest::GTest ) diff --git a/storage/src/tests/persistence/filestorage/CMakeLists.txt b/storage/src/tests/persistence/filestorage/CMakeLists.txt index c9623e991e0..00a57410b54 100644 --- a/storage/src/tests/persistence/filestorage/CMakeLists.txt +++ b/storage/src/tests/persistence/filestorage/CMakeLists.txt @@ -15,7 +15,6 @@ vespa_add_executable(storage_filestorage_gtest_runner_app TEST gtest_runner.cpp DEPENDS storage - storageapi storage_testhostreporter storage_testpersistence_common GTest::GTest diff --git a/storage/src/tests/persistence/splitbitdetectortest.cpp b/storage/src/tests/persistence/splitbitdetectortest.cpp index 86575c75b60..b93783f5cdf 100644 --- a/storage/src/tests/persistence/splitbitdetectortest.cpp +++ b/storage/src/tests/persistence/splitbitdetectortest.cpp @@ -162,46 +162,4 @@ TEST_F(SplitBitDetectorTest, zero_doc_limit_falls_back_to_one_bit_increase_on_gi result.toString()); } -/** - * Not a regular unit test in itself, but more of an utility to find non-unique - * document IDs that map to the same 58-bit bucket ID. Disabled by default since - * it costs CPU to do this and is not necessary during normal testing. - */ -TEST_F(SplitBitDetectorTest, DISABLED_find_bucket_collision_ids) { - using document::DocumentId; - using document::BucketId; - - document::BucketIdFactory factory; - - DocumentId targetId("id:foo:music:n=123456:ABCDEFGHIJKLMN"); - BucketId targetBucket(factory.getBucketId(targetId)); - char candidateSuffix[] = "ABCDEFGHIJKLMN"; - - size_t iterations = 0; - constexpr size_t maxIterations = 100000000; - while (std::next_permutation(std::begin(candidateSuffix), std::end(candidateSuffix) - 1)) - { - ++iterations; - - DocumentId candidateId(vespalib::make_string("id:foo:music:n=123456:%s",candidateSuffix)); - BucketId candidateBucket(factory.getBucketId(candidateId)); - if (targetBucket == candidateBucket) { - std::cerr << "\nFound a collision after " << iterations - << " iterations!\n" - << "target: " << targetId << " -> " << targetBucket - << "\ncollision: " << candidateId << " -> " - << candidateBucket << "\n"; - return; - } - - if (iterations == maxIterations) { - std::cerr << "\nNo collision found after " << iterations - << " iterations :[\n"; - return; - } - } - std::cerr << "\nRan out of permutations after " << iterations - << " iterations!\n"; -} - } diff --git a/storageapi/src/tests/.gitignore b/storage/src/tests/storageapi/.gitignore index 29911acfe67..29911acfe67 100644 --- a/storageapi/src/tests/.gitignore +++ b/storage/src/tests/storageapi/.gitignore diff --git a/storageapi/src/tests/CMakeLists.txt b/storage/src/tests/storageapi/CMakeLists.txt index 18d2b34f499..cfcdfd55350 100644 --- a/storageapi/src/tests/CMakeLists.txt +++ b/storage/src/tests/storageapi/CMakeLists.txt @@ -7,7 +7,7 @@ vespa_add_executable(storageapi_gtest_runner_app TEST storageapi_testbuckets storageapi_testmbusprot storageapi_testmessageapi - storageapi + storage GTest::GTest ) diff --git a/storageapi/src/tests/buckets/.gitignore b/storage/src/tests/storageapi/buckets/.gitignore index 1d859456ef9..1d859456ef9 100644 --- a/storageapi/src/tests/buckets/.gitignore +++ b/storage/src/tests/storageapi/buckets/.gitignore diff --git a/storageapi/src/tests/buckets/CMakeLists.txt b/storage/src/tests/storageapi/buckets/CMakeLists.txt index 651b27618a6..7d2bdb77557 100644 --- a/storageapi/src/tests/buckets/CMakeLists.txt +++ b/storage/src/tests/storageapi/buckets/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageapi_testbuckets SOURCES bucketinfotest.cpp DEPENDS - storageapi + storage GTest::GTest ) diff --git a/storageapi/src/tests/buckets/bucketinfotest.cpp b/storage/src/tests/storageapi/buckets/bucketinfotest.cpp index 25648fa7e96..25648fa7e96 100644 --- a/storageapi/src/tests/buckets/bucketinfotest.cpp +++ b/storage/src/tests/storageapi/buckets/bucketinfotest.cpp diff --git a/storageapi/src/tests/gtest_runner.cpp b/storage/src/tests/storageapi/gtest_runner.cpp index 2ae414830b9..2ae414830b9 100644 --- a/storageapi/src/tests/gtest_runner.cpp +++ b/storage/src/tests/storageapi/gtest_runner.cpp diff --git a/storageapi/src/tests/mbusprot/.gitignore b/storage/src/tests/storageapi/mbusprot/.gitignore index 1b779fa320a..1b779fa320a 100644 --- a/storageapi/src/tests/mbusprot/.gitignore +++ b/storage/src/tests/storageapi/mbusprot/.gitignore diff --git a/storageapi/src/tests/mbusprot/CMakeLists.txt b/storage/src/tests/storageapi/mbusprot/CMakeLists.txt index 48469cfd9ea..1f10f6d519a 100644 --- a/storageapi/src/tests/mbusprot/CMakeLists.txt +++ b/storage/src/tests/storageapi/mbusprot/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageapi_testmbusprot SOURCES storageprotocoltest.cpp DEPENDS - storageapi + storage GTest::GTest ) diff --git a/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE b/storage/src/tests/storageapi/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE index 5045a98b037..5045a98b037 100644 --- a/storageapi/src/tests/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE +++ b/storage/src/tests/storageapi/mbusprot/mbusprot.4.2.serialization.V_4_2_STABLE diff --git a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp index c130e433285..c130e433285 100644 --- a/storageapi/src/tests/mbusprot/storageprotocoltest.cpp +++ b/storage/src/tests/storageapi/mbusprot/storageprotocoltest.cpp diff --git a/storageapi/src/tests/message/.gitignore b/storage/src/tests/storageapi/messageapi/.gitignore index 1d859456ef9..1d859456ef9 100644 --- a/storageapi/src/tests/message/.gitignore +++ b/storage/src/tests/storageapi/messageapi/.gitignore diff --git a/storageapi/src/tests/messageapi/CMakeLists.txt b/storage/src/tests/storageapi/messageapi/CMakeLists.txt index 5bc89dea4fc..4866aa63079 100644 --- a/storageapi/src/tests/messageapi/CMakeLists.txt +++ b/storage/src/tests/storageapi/messageapi/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageapi_testmessageapi SOURCES storage_message_address_test.cpp DEPENDS - storageapi + storage GTest::GTest ) diff --git a/storageapi/src/tests/messageapi/storage_message_address_test.cpp b/storage/src/tests/storageapi/messageapi/storage_message_address_test.cpp index ea59fefc924..ea59fefc924 100644 --- a/storageapi/src/tests/messageapi/storage_message_address_test.cpp +++ b/storage/src/tests/storageapi/messageapi/storage_message_address_test.cpp diff --git a/storageframework/src/tests/.gitignore b/storage/src/tests/storageframework/.gitignore index 94b0b80fe52..94b0b80fe52 100644 --- a/storageframework/src/tests/.gitignore +++ b/storage/src/tests/storageframework/.gitignore diff --git a/storageframework/src/tests/CMakeLists.txt b/storage/src/tests/storageframework/CMakeLists.txt index fdda595125a..fdda595125a 100644 --- a/storageframework/src/tests/CMakeLists.txt +++ b/storage/src/tests/storageframework/CMakeLists.txt diff --git a/searchcorespi/src/vespa/searchcorespi/flush/.gitignore b/storage/src/tests/storageframework/clock/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/searchcorespi/src/vespa/searchcorespi/flush/.gitignore +++ b/storage/src/tests/storageframework/clock/.gitignore diff --git a/storageframework/src/tests/clock/CMakeLists.txt b/storage/src/tests/storageframework/clock/CMakeLists.txt index 46015e48cb2..541b70d121a 100644 --- a/storageframework/src/tests/clock/CMakeLists.txt +++ b/storage/src/tests/storageframework/clock/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_library(storageframework_testclock SOURCES timetest.cpp DEPENDS - storageframework + storage GTest::GTest ) diff --git a/storageframework/src/tests/clock/timetest.cpp b/storage/src/tests/storageframework/clock/timetest.cpp index e1270156aa0..e1270156aa0 100644 --- a/storageframework/src/tests/clock/timetest.cpp +++ b/storage/src/tests/storageframework/clock/timetest.cpp diff --git a/storageframework/src/tests/gtest_runner.cpp b/storage/src/tests/storageframework/gtest_runner.cpp index 105477e0b4b..105477e0b4b 100644 --- a/storageframework/src/tests/gtest_runner.cpp +++ b/storage/src/tests/storageframework/gtest_runner.cpp diff --git a/searchcorespi/src/vespa/searchcorespi/index/.gitignore b/storage/src/tests/storageframework/thread/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/searchcorespi/src/vespa/searchcorespi/index/.gitignore +++ b/storage/src/tests/storageframework/thread/.gitignore diff --git a/storageframework/src/tests/thread/CMakeLists.txt b/storage/src/tests/storageframework/thread/CMakeLists.txt index d50f5933c51..1b351bb15eb 100644 --- a/storageframework/src/tests/thread/CMakeLists.txt +++ b/storage/src/tests/storageframework/thread/CMakeLists.txt @@ -4,6 +4,6 @@ vespa_add_library(storageframework_testthread tickingthreadtest.cpp taskthreadtest.cpp DEPENDS - storageframework + storage GTest::GTest ) diff --git a/storageframework/src/tests/thread/taskthreadtest.cpp b/storage/src/tests/storageframework/thread/taskthreadtest.cpp index 32bc6e8a8fb..32bc6e8a8fb 100644 --- a/storageframework/src/tests/thread/taskthreadtest.cpp +++ b/storage/src/tests/storageframework/thread/taskthreadtest.cpp diff --git a/storageframework/src/tests/thread/tickingthreadtest.cpp b/storage/src/tests/storageframework/thread/tickingthreadtest.cpp index 577e9d128b9..577e9d128b9 100644 --- a/storageframework/src/tests/thread/tickingthreadtest.cpp +++ b/storage/src/tests/storageframework/thread/tickingthreadtest.cpp diff --git a/storage/src/tests/storageserver/CMakeLists.txt b/storage/src/tests/storageserver/CMakeLists.txt index 4e0b9a2c4de..22d3e71021c 100644 --- a/storage/src/tests/storageserver/CMakeLists.txt +++ b/storage/src/tests/storageserver/CMakeLists.txt @@ -3,8 +3,8 @@ vespa_add_library(storage_teststorageserver TEST SOURCES testvisitormessagesession.cpp DEPENDS - storage_storageserver storage_testcommon + storage ) vespa_add_executable(storage_storageserver_gtest_runner_app TEST @@ -21,9 +21,9 @@ vespa_add_executable(storage_storageserver_gtest_runner_app TEST statereportertest.cpp gtest_runner.cpp DEPENDS - storage_storageserver storage_testcommon storage_teststorageserver + storage GTest::GTest ) diff --git a/storage/src/tests/storageserver/rpc/CMakeLists.txt b/storage/src/tests/storageserver/rpc/CMakeLists.txt index db1ffd3f77d..4e5025a58d2 100644 --- a/storage/src/tests/storageserver/rpc/CMakeLists.txt +++ b/storage/src/tests/storageserver/rpc/CMakeLists.txt @@ -7,9 +7,9 @@ vespa_add_executable(storage_storageserver_rpc_gtest_runner_app TEST storage_api_rpc_service_test.cpp gtest_runner.cpp DEPENDS - storage_storageserver storage_testcommon storage_teststorageserver + storage GTest::GTest ) diff --git a/storage/src/vespa/storage/CMakeLists.txt b/storage/src/vespa/storage/CMakeLists.txt index d7fca273229..3cb9358b56d 100644 --- a/storage/src/vespa/storage/CMakeLists.txt +++ b/storage/src/vespa/storage/CMakeLists.txt @@ -12,6 +12,27 @@ vespa_add_library(storage $<TARGET_OBJECTS:storage_thread> $<TARGET_OBJECTS:storage_status> $<TARGET_OBJECTS:storage_component> + $<TARGET_OBJECTS:storage_storageserver_rpc> + $<TARGET_OBJECTS:storage_storageserver> + $<TARGET_OBJECTS:storage_distributoroperation> + $<TARGET_OBJECTS:storage_distributoroperationexternal> + $<TARGET_OBJECTS:storage_distributoroperationidealstate> + $<TARGET_OBJECTS:storage_distributormaintenance> + $<TARGET_OBJECTS:storage_distributor_bucketdb> + $<TARGET_OBJECTS:storage_distributor> + $<TARGET_OBJECTS:storageframework_component> + $<TARGET_OBJECTS:storageframework_status> + $<TARGET_OBJECTS:storageframework_thread> + $<TARGET_OBJECTS:storageframework_clock> + $<TARGET_OBJECTS:storageframework_clockimpl> + $<TARGET_OBJECTS:storageframework_componentimpl> + $<TARGET_OBJECTS:storageframework_threadimpl> + $<TARGET_OBJECTS:storageapi_message> + $<TARGET_OBJECTS:storageapi_buckets> + $<TARGET_OBJECTS:storageapi_messageapi> + $<TARGET_OBJECTS:storageapi_mbusprot> INSTALL lib64 DEPENDS ) + +vespa_add_target_package_dependency(storage Protobuf) diff --git a/storage/src/vespa/storage/distributor/CMakeLists.txt b/storage/src/vespa/storage/distributor/CMakeLists.txt index 470bfb69abb..184fee5d2c9 100644 --- a/storage/src/vespa/storage/distributor/CMakeLists.txt +++ b/storage/src/vespa/storage/distributor/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_distributor +vespa_add_library(storage_distributor OBJECT SOURCES activecopy.cpp blockingoperationstarter.cpp @@ -55,12 +55,5 @@ vespa_add_library(storage_distributor top_level_distributor.cpp update_metric_set.cpp visitormetricsset.cpp - $<TARGET_OBJECTS:storage_distributoroperation> - $<TARGET_OBJECTS:storage_distributoroperationexternal> - $<TARGET_OBJECTS:storage_distributoroperationidealstate> - $<TARGET_OBJECTS:storage_distributormaintenance> - $<TARGET_OBJECTS:storage_distributor_bucketdb> - INSTALL lib64 DEPENDS - storage ) diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp index 8b172743d27..30d13e4eb1a 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_pool.cpp @@ -117,6 +117,7 @@ void DistributorStripePool::start(const std::vector<TickableStripe*>& stripes) { if (_single_threaded_test_mode) { return; // We want all the control structures in place, but none of the actual OS threads. } + std::unique_lock lock(_mutex); // Ensure _threads is visible to all started threads for (auto& s : _stripes) { _threads.emplace_back(_thread_pool.NewThread(s.get())); } diff --git a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp index ae5445da620..8f37dbbbf5d 100644 --- a/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp +++ b/storage/src/vespa/storage/distributor/distributor_stripe_thread.cpp @@ -47,7 +47,7 @@ void DistributorStripeThread::signal_wants_park() noexcept { assert(!should_park_relaxed()); _should_park.store(true, std::memory_order_relaxed); if (_waiting_for_event) { - _event_cond.notify_one(); // TODO after unlock? + _event_cond.notify_one(); } } @@ -55,7 +55,7 @@ void DistributorStripeThread::unpark_thread() noexcept { std::lock_guard lock(_mutex); assert(should_park_relaxed()); _should_park.store(false, std::memory_order_relaxed); - _park_cond.notify_one(); // TODO after unlock? + _park_cond.notify_one(); } void DistributorStripeThread::wait_until_event_notified_or_timed_out() noexcept { diff --git a/storage/src/vespa/storage/persistence/bucketprocessor.h b/storage/src/vespa/storage/persistence/bucketprocessor.h index 002cba6c15c..6338b447f7b 100644 --- a/storage/src/vespa/storage/persistence/bucketprocessor.h +++ b/storage/src/vespa/storage/persistence/bucketprocessor.h @@ -8,7 +8,7 @@ #include <vespa/persistence/spi/bucket.h> #include <vespa/persistence/spi/context.h> -#include <persistence/spi/types.h> +#include <vespa/persistence/spi/types.h> namespace document { class FieldSet; } namespace storage::spi { diff --git a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp index c44ae305fa2..cfcc32a76e5 100644 --- a/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp +++ b/storage/src/vespa/storage/persistence/filestorage/filestorhandlerimpl.cpp @@ -78,7 +78,10 @@ FileStorHandlerImpl::FileStorHandlerImpl(uint32_t numThreads, uint32_t numStripe _component.registerMetricUpdateHook(*this, framework::SecondTime(5)); } -FileStorHandlerImpl::~FileStorHandlerImpl() = default; +FileStorHandlerImpl::~FileStorHandlerImpl() +{ + waitUntilNoLocks(); +} void FileStorHandlerImpl::addMergeStatus(const document::Bucket& bucket, std::shared_ptr<MergeStatus> status) @@ -170,8 +173,13 @@ FileStorHandlerImpl::flush(bool killPendingMerges) LOG(debug, "All queues and bucket locks released."); if (killPendingMerges) { + std::map<document::Bucket, std::shared_ptr<MergeStatus>> my_merge_states; + { + std::lock_guard mergeGuard(_mergeStatesLock); + std::swap(_mergeStates, my_merge_states); + } api::ReturnCode code(api::ReturnCode::ABORTED, "Storage node is shutting down"); - for (auto & entry : _mergeStates) { + for (auto & entry : my_merge_states) { MergeStatus& s(*entry.second); if (s.pendingGetDiff) { s.pendingGetDiff->setResult(code); @@ -186,7 +194,6 @@ FileStorHandlerImpl::flush(bool killPendingMerges) _messageSender.sendReply(s.reply); } } - _mergeStates.clear(); } } @@ -1171,7 +1178,6 @@ FileStorHandlerImpl::Stripe::release(const document::Bucket & bucket, if (!entry._exclusiveLock && entry._sharedLocks.empty()) { _lockedBuckets.erase(iter); // No more locks held } - guard.unlock(); _cond->notify_all(); } diff --git a/storage/src/vespa/storage/storageserver/CMakeLists.txt b/storage/src/vespa/storage/storageserver/CMakeLists.txt index 1706b858256..8b82bb251b2 100644 --- a/storage/src/vespa/storage/storageserver/CMakeLists.txt +++ b/storage/src/vespa/storage/storageserver/CMakeLists.txt @@ -1,6 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storage_storageserver +vespa_add_library(storage_storageserver OBJECT SOURCES bouncer.cpp bouncer_metrics.cpp @@ -28,8 +28,5 @@ vespa_add_library(storage_storageserver storagenodecontext.cpp tls_statistics_metrics_wrapper.cpp $<TARGET_OBJECTS:storage_storageserver_rpc> - INSTALL lib64 DEPENDS - storage - storage_distributor ) diff --git a/storage/src/vespa/storage/storageserver/bouncer.cpp b/storage/src/vespa/storage/storageserver/bouncer.cpp index 2b02156367c..313773bc5e8 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.cpp +++ b/storage/src/vespa/storage/storageserver/bouncer.cpp @@ -145,22 +145,22 @@ Bouncer::rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, } void -Bouncer::abortCommandDueToClusterDown(api::StorageMessage& msg) +Bouncer::abortCommandDueToClusterDown(api::StorageMessage& msg, const lib::State& cluster_state) { std::shared_ptr<api::StorageReply> reply( static_cast<api::StorageCommand&>(msg).makeReply().release()); std::ostringstream ost; ost << "We don't allow external load while cluster is in state " - << _clusterState->toString(true); + << cluster_state.toString(true); append_node_identity(ost); reply->setResult(api::ReturnCode(api::ReturnCode::ABORTED, ost.str())); sendUp(reply); } bool -Bouncer::clusterIsUp() const +Bouncer::clusterIsUp(const lib::State& cluster_state) { - return (*_clusterState == lib::State::UP); + return (cluster_state == lib::State::UP); } bool Bouncer::isDistributor() const { @@ -264,14 +264,14 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) int maxClockSkewInSeconds; bool isInAvailableState; bool abortLoadWhenClusterDown; - bool cluster_is_up; + const lib::State *cluster_state; int feedPriorityLowerBound; { std::lock_guard lock(_lock); state = &getDerivedNodeState(msg->getBucket().getBucketSpace()).getState(); maxClockSkewInSeconds = _config->maxClockSkewSeconds; abortLoadWhenClusterDown = _config->stopExternalLoadWhenClusterDown; - cluster_is_up = clusterIsUp(); + cluster_state = _clusterState; isInAvailableState = state->oneOf(_config->stopAllLoadWhenNodestateNotIn.c_str()); feedPriorityLowerBound = _config->feedRejectionPriorityThreshold; } @@ -285,7 +285,7 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) } // Special case for point lookup Gets while node is in maintenance mode // to allow reads to complete during two-phase cluster state transitions - if ((*state == lib::State::MAINTENANCE) && (type.getId() == api::MessageType::GET_ID) && clusterIsUp()) { + if ((*state == lib::State::MAINTENANCE) && (type.getId() == api::MessageType::GET_ID) && clusterIsUp(*cluster_state)) { MBUS_TRACE(msg->getTrace(), 7, "Bouncer: node is in Maintenance mode, but letting Get through"); return false; } @@ -319,8 +319,8 @@ Bouncer::onDown(const std::shared_ptr<api::StorageMessage>& msg) } // If cluster state is not up, fail external load - if (abortLoadWhenClusterDown && !cluster_is_up) { - abortCommandDueToClusterDown(*msg); + if (abortLoadWhenClusterDown && !clusterIsUp(*cluster_state)) { + abortCommandDueToClusterDown(*msg, *cluster_state); return true; } diff --git a/storage/src/vespa/storage/storageserver/bouncer.h b/storage/src/vespa/storage/storageserver/bouncer.h index 50a08387b39..55fb386a37e 100644 --- a/storage/src/vespa/storage/storageserver/bouncer.h +++ b/storage/src/vespa/storage/storageserver/bouncer.h @@ -57,10 +57,10 @@ private: void onClose() override; void abortCommandForUnavailableNode(api::StorageMessage&, const lib::State&); void rejectCommandWithTooHighClockSkew(api::StorageMessage& msg, int maxClockSkewInSeconds); - void abortCommandDueToClusterDown(api::StorageMessage&); + void abortCommandDueToClusterDown(api::StorageMessage&, const lib::State&); void rejectDueToInsufficientPriority(api::StorageMessage&, api::StorageMessage::Priority); void reject_due_to_too_few_bucket_bits(api::StorageMessage&); - bool clusterIsUp() const; + static bool clusterIsUp(const lib::State& cluster_state); bool isDistributor() const; bool isExternalLoad(const api::MessageType&) const noexcept; bool isExternalWriteOperation(const api::MessageType&) const noexcept; diff --git a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp index 7ad59ee574c..c98617c84bf 100644 --- a/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp +++ b/storage/src/vespa/storage/storageserver/rpc/shared_rpc_resources.cpp @@ -115,7 +115,6 @@ void SharedRpcResources::shutdown() { std::this_thread::sleep_for(10ms); } } - _slobrok_register.reset(); // Implicitly kill any pending slobrok tasks prior to shutting down transport layer _transport->ShutDown(true); // FIXME need to reset to break weak_ptrs? But ShutDown should already sync pending resolves...! _shutdown = true; diff --git a/storage/src/vespa/storage/storageserver/statereporter.h b/storage/src/vespa/storage/storageserver/statereporter.h index 71ac1f3fdfd..25ad6814dad 100644 --- a/storage/src/vespa/storage/storageserver/statereporter.h +++ b/storage/src/vespa/storage/storageserver/statereporter.h @@ -14,8 +14,8 @@ #include <vespa/storage/common/storagecomponent.h> #include <vespa/storageframework/generic/status/statusreporter.h> #include <vespa/metrics/state_api_adapter.h> -#include <vespa/vespalib/net/metrics_producer.h> -#include <vespa/vespalib/net/state_api.h> +#include <vespa/vespalib/net/http/metrics_producer.h> +#include <vespa/vespalib/net/http/state_api.h> namespace vespalib { class StringTokenizer; diff --git a/storageapi/src/vespa/storageapi/.gitignore b/storage/src/vespa/storageapi/.gitignore index 43485abf58c..43485abf58c 100644 --- a/storageapi/src/vespa/storageapi/.gitignore +++ b/storage/src/vespa/storageapi/.gitignore diff --git a/storageapi/src/vespa/storageapi/app/.gitignore b/storage/src/vespa/storageapi/app/.gitignore index fa917bb5ae5..fa917bb5ae5 100644 --- a/storageapi/src/vespa/storageapi/app/.gitignore +++ b/storage/src/vespa/storageapi/app/.gitignore diff --git a/storageapi/src/vespa/storageapi/app/CMakeLists.txt b/storage/src/vespa/storageapi/app/CMakeLists.txt index 90e5567d705..a8c183b01de 100644 --- a/storageapi/src/vespa/storageapi/app/CMakeLists.txt +++ b/storage/src/vespa/storageapi/app/CMakeLists.txt @@ -3,5 +3,5 @@ vespa_add_executable(storageapi_getbucketid_app SOURCES getbucketid.cpp DEPENDS - storageapi + storage ) diff --git a/storageapi/src/vespa/storageapi/app/getbucketid.cpp b/storage/src/vespa/storageapi/app/getbucketid.cpp index 21f7912d1a1..21f7912d1a1 100644 --- a/storageapi/src/vespa/storageapi/app/getbucketid.cpp +++ b/storage/src/vespa/storageapi/app/getbucketid.cpp diff --git a/storageapi/src/vespa/storageapi/buckets/.gitignore b/storage/src/vespa/storageapi/buckets/.gitignore index 7ad21cf1c5e..7ad21cf1c5e 100644 --- a/storageapi/src/vespa/storageapi/buckets/.gitignore +++ b/storage/src/vespa/storageapi/buckets/.gitignore diff --git a/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt b/storage/src/vespa/storageapi/buckets/CMakeLists.txt index ee087f6f566..ee087f6f566 100644 --- a/storageapi/src/vespa/storageapi/buckets/CMakeLists.txt +++ b/storage/src/vespa/storageapi/buckets/CMakeLists.txt diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp b/storage/src/vespa/storageapi/buckets/bucketinfo.cpp index ff2f40e736b..8305b999afc 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.cpp +++ b/storage/src/vespa/storageapi/buckets/bucketinfo.cpp @@ -2,7 +2,6 @@ #include "bucketinfo.h" #include <vespa/vespalib/stllike/asciistream.h> #include <vespa/vespalib/util/xmlstream.h> -#include <ostream> namespace storage::api { @@ -111,7 +110,7 @@ BucketInfo::toString() const } void -BucketInfo::printXml(vespalib::XmlOutputStream& xos) const +BucketInfo::printXml(vespalib::xml::XmlOutputStream& xos) const { using namespace vespalib::xml; xos << XmlAttribute("checksum", _checksum, XmlAttribute::HEX) diff --git a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h b/storage/src/vespa/storageapi/buckets/bucketinfo.h index d7b407185f6..e535dee3152 100644 --- a/storageapi/src/vespa/storageapi/buckets/bucketinfo.h +++ b/storage/src/vespa/storageapi/buckets/bucketinfo.h @@ -14,9 +14,9 @@ #pragma once #include <vespa/storageapi/defs.h> -#include <vespa/vespalib/util/xmlserializable.h> #include <vespa/vespalib/stllike/string.h> +namespace vespalib::xml { class XmlOutputStream; } namespace storage::api { class BucketInfo @@ -76,7 +76,7 @@ public: return _metaCount == 0 && _usedFileSize == 0 && _checksum == 0; } vespalib::string toString() const; - void printXml(vespalib::XmlOutputStream&) const; + void printXml(vespalib::xml::XmlOutputStream&) const; }; std::ostream & operator << (std::ostream & os, const BucketInfo & bucketInfo); diff --git a/storageapi/src/vespa/storageapi/defs.h b/storage/src/vespa/storageapi/defs.h index 9ee9fdf2218..9ee9fdf2218 100644 --- a/storageapi/src/vespa/storageapi/defs.h +++ b/storage/src/vespa/storageapi/defs.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/.gitignore b/storage/src/vespa/storageapi/mbusprot/.gitignore index 8e91fe9cab0..8e91fe9cab0 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/.gitignore +++ b/storage/src/vespa/storageapi/mbusprot/.gitignore diff --git a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt b/storage/src/vespa/storageapi/mbusprot/CMakeLists.txt index 43c1da32502..43c1da32502 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/CMakeLists.txt +++ b/storage/src/vespa/storageapi/mbusprot/CMakeLists.txt diff --git a/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h b/storage/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h index f3c6a6f6856..f3c6a6f6856 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h +++ b/storage/src/vespa/storageapi/mbusprot/legacyprotocolserialization.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h b/storage/src/vespa/storageapi/mbusprot/oldreturncodemapper.h index 40e09f23697..40e09f23697 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/oldreturncodemapper.h +++ b/storage/src/vespa/storageapi/mbusprot/oldreturncodemapper.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/common.proto index 49e1a8f8aba..49e1a8f8aba 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/common.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/common.proto diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto index cb923db3c3c..cb923db3c3c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/feed.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/feed.proto diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/inspect.proto index c3f4b1263a1..c3f4b1263a1 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/inspect.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/inspect.proto diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto index 7f7ab1d7c0b..7f7ab1d7c0b 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/maintenance.proto diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto b/storage/src/vespa/storageapi/mbusprot/protobuf/visiting.proto index 35d69bc2d3e..35d69bc2d3e 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf/visiting.proto +++ b/storage/src/vespa/storageapi/mbusprot/protobuf/visiting.proto diff --git a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h b/storage/src/vespa/storageapi/mbusprot/protobuf_includes.h index 9accfdf75ee..9accfdf75ee 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protobuf_includes.h +++ b/storage/src/vespa/storageapi/mbusprot/protobuf_includes.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization.cpp index 4614659c458..4614659c458 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization.h index 653e9ded85a..653e9ded85a 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp index 1bea0ea74c9..1bea0ea74c9 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization4_2.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization4_2.h index b8a20e9e401..b8a20e9e401 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization4_2.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization4_2.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp index fa400b565b2..fa400b565b2 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.h index 8de2edab17f..8de2edab17f 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_0.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_0.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp index 97ceceac33d..97ceceac33d 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_1.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_1.h index bfad492e653..bfad492e653 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_1.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_1.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp index 10aa9d69fbc..10aa9d69fbc 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_2.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_2.h index e56b9942fa5..e56b9942fa5 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization5_2.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization5_2.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp index 799aff8f8b3..799aff8f8b3 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization6_0.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization6_0.h index 5467cf6c5d2..5467cf6c5d2 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization6_0.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization6_0.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp index 91b5999e34c..91b5999e34c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.cpp +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.h index a61397c85ac..a61397c85ac 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/protocolserialization7.h +++ b/storage/src/vespa/storageapi/mbusprot/protocolserialization7.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h index 457a6178704..457a6178704 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/serializationhelper.h +++ b/storage/src/vespa/storageapi/mbusprot/serializationhelper.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp b/storage/src/vespa/storageapi/mbusprot/storagecommand.cpp index 34fd0992adb..34fd0992adb 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.cpp +++ b/storage/src/vespa/storageapi/mbusprot/storagecommand.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h b/storage/src/vespa/storageapi/mbusprot/storagecommand.h index e65c0295e31..e65c0295e31 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagecommand.h +++ b/storage/src/vespa/storageapi/mbusprot/storagecommand.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp b/storage/src/vespa/storageapi/mbusprot/storagemessage.cpp index e69de29bb2d..e69de29bb2d 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.cpp +++ b/storage/src/vespa/storageapi/mbusprot/storagemessage.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h b/storage/src/vespa/storageapi/mbusprot/storagemessage.h index 61323222b89..61323222b89 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagemessage.h +++ b/storage/src/vespa/storageapi/mbusprot/storagemessage.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp b/storage/src/vespa/storageapi/mbusprot/storageprotocol.cpp index 3cf54860f7c..3cf54860f7c 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.cpp +++ b/storage/src/vespa/storageapi/mbusprot/storageprotocol.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h b/storage/src/vespa/storageapi/mbusprot/storageprotocol.h index 3f36ac42117..3f36ac42117 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storageprotocol.h +++ b/storage/src/vespa/storageapi/mbusprot/storageprotocol.h diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp b/storage/src/vespa/storageapi/mbusprot/storagereply.cpp index 1db6912dd33..1db6912dd33 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.cpp +++ b/storage/src/vespa/storageapi/mbusprot/storagereply.cpp diff --git a/storageapi/src/vespa/storageapi/mbusprot/storagereply.h b/storage/src/vespa/storageapi/mbusprot/storagereply.h index 538b7caa678..538b7caa678 100644 --- a/storageapi/src/vespa/storageapi/mbusprot/storagereply.h +++ b/storage/src/vespa/storageapi/mbusprot/storagereply.h diff --git a/storageapi/src/vespa/storageapi/message/.gitignore b/storage/src/vespa/storageapi/message/.gitignore index 526f91c6668..526f91c6668 100644 --- a/storageapi/src/vespa/storageapi/message/.gitignore +++ b/storage/src/vespa/storageapi/message/.gitignore diff --git a/storageapi/src/vespa/storageapi/message/CMakeLists.txt b/storage/src/vespa/storageapi/message/CMakeLists.txt index 2a761921dff..2a761921dff 100644 --- a/storageapi/src/vespa/storageapi/message/CMakeLists.txt +++ b/storage/src/vespa/storageapi/message/CMakeLists.txt diff --git a/storageapi/src/vespa/storageapi/message/bucket.cpp b/storage/src/vespa/storageapi/message/bucket.cpp index 520f1aa2741..520f1aa2741 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.cpp +++ b/storage/src/vespa/storageapi/message/bucket.cpp diff --git a/storageapi/src/vespa/storageapi/message/bucket.h b/storage/src/vespa/storageapi/message/bucket.h index 47785a92039..47785a92039 100644 --- a/storageapi/src/vespa/storageapi/message/bucket.h +++ b/storage/src/vespa/storageapi/message/bucket.h diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp b/storage/src/vespa/storageapi/message/bucketsplitting.cpp index 784ba6edbd1..784ba6edbd1 100644 --- a/storageapi/src/vespa/storageapi/message/bucketsplitting.cpp +++ b/storage/src/vespa/storageapi/message/bucketsplitting.cpp diff --git a/storageapi/src/vespa/storageapi/message/bucketsplitting.h b/storage/src/vespa/storageapi/message/bucketsplitting.h index 584cdfd5638..584cdfd5638 100644 --- a/storageapi/src/vespa/storageapi/message/bucketsplitting.h +++ b/storage/src/vespa/storageapi/message/bucketsplitting.h diff --git a/storageapi/src/vespa/storageapi/message/datagram.cpp b/storage/src/vespa/storageapi/message/datagram.cpp index 546e0edecc1..546e0edecc1 100644 --- a/storageapi/src/vespa/storageapi/message/datagram.cpp +++ b/storage/src/vespa/storageapi/message/datagram.cpp diff --git a/storageapi/src/vespa/storageapi/message/datagram.h b/storage/src/vespa/storageapi/message/datagram.h index e0f5a9f7b30..e0f5a9f7b30 100644 --- a/storageapi/src/vespa/storageapi/message/datagram.h +++ b/storage/src/vespa/storageapi/message/datagram.h diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.cpp b/storage/src/vespa/storageapi/message/documentsummary.cpp index 6909b4d223c..6909b4d223c 100644 --- a/storageapi/src/vespa/storageapi/message/documentsummary.cpp +++ b/storage/src/vespa/storageapi/message/documentsummary.cpp diff --git a/storageapi/src/vespa/storageapi/message/documentsummary.h b/storage/src/vespa/storageapi/message/documentsummary.h index 5e2c1af3cfd..5e2c1af3cfd 100644 --- a/storageapi/src/vespa/storageapi/message/documentsummary.h +++ b/storage/src/vespa/storageapi/message/documentsummary.h diff --git a/storageapi/src/vespa/storageapi/message/internal.cpp b/storage/src/vespa/storageapi/message/internal.cpp index 5e1daaaf3be..5e1daaaf3be 100644 --- a/storageapi/src/vespa/storageapi/message/internal.cpp +++ b/storage/src/vespa/storageapi/message/internal.cpp diff --git a/storageapi/src/vespa/storageapi/message/internal.h b/storage/src/vespa/storageapi/message/internal.h index e0fee3e5494..e0fee3e5494 100644 --- a/storageapi/src/vespa/storageapi/message/internal.h +++ b/storage/src/vespa/storageapi/message/internal.h diff --git a/storageapi/src/vespa/storageapi/message/persistence.cpp b/storage/src/vespa/storageapi/message/persistence.cpp index 41a53449b67..41a53449b67 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.cpp +++ b/storage/src/vespa/storageapi/message/persistence.cpp diff --git a/storageapi/src/vespa/storageapi/message/persistence.h b/storage/src/vespa/storageapi/message/persistence.h index 1d8cfd9d277..1d8cfd9d277 100644 --- a/storageapi/src/vespa/storageapi/message/persistence.h +++ b/storage/src/vespa/storageapi/message/persistence.h diff --git a/storageapi/src/vespa/storageapi/message/queryresult.cpp b/storage/src/vespa/storageapi/message/queryresult.cpp index b5d29cf4e02..b5d29cf4e02 100644 --- a/storageapi/src/vespa/storageapi/message/queryresult.cpp +++ b/storage/src/vespa/storageapi/message/queryresult.cpp diff --git a/storageapi/src/vespa/storageapi/message/queryresult.h b/storage/src/vespa/storageapi/message/queryresult.h index c3bbbdc47ce..c3bbbdc47ce 100644 --- a/storageapi/src/vespa/storageapi/message/queryresult.h +++ b/storage/src/vespa/storageapi/message/queryresult.h diff --git a/storageapi/src/vespa/storageapi/message/removelocation.cpp b/storage/src/vespa/storageapi/message/removelocation.cpp index 7b7ed894b2c..7b7ed894b2c 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.cpp +++ b/storage/src/vespa/storageapi/message/removelocation.cpp diff --git a/storageapi/src/vespa/storageapi/message/removelocation.h b/storage/src/vespa/storageapi/message/removelocation.h index 276b090cc57..276b090cc57 100644 --- a/storageapi/src/vespa/storageapi/message/removelocation.h +++ b/storage/src/vespa/storageapi/message/removelocation.h diff --git a/storageapi/src/vespa/storageapi/message/searchresult.cpp b/storage/src/vespa/storageapi/message/searchresult.cpp index b2cf04b0410..b2cf04b0410 100644 --- a/storageapi/src/vespa/storageapi/message/searchresult.cpp +++ b/storage/src/vespa/storageapi/message/searchresult.cpp diff --git a/storageapi/src/vespa/storageapi/message/searchresult.h b/storage/src/vespa/storageapi/message/searchresult.h index 4795876102d..4795876102d 100644 --- a/storageapi/src/vespa/storageapi/message/searchresult.h +++ b/storage/src/vespa/storageapi/message/searchresult.h diff --git a/storageapi/src/vespa/storageapi/message/stat.cpp b/storage/src/vespa/storageapi/message/stat.cpp index 3b97f4f5541..3b97f4f5541 100644 --- a/storageapi/src/vespa/storageapi/message/stat.cpp +++ b/storage/src/vespa/storageapi/message/stat.cpp diff --git a/storageapi/src/vespa/storageapi/message/stat.h b/storage/src/vespa/storageapi/message/stat.h index 0797ae43799..0797ae43799 100644 --- a/storageapi/src/vespa/storageapi/message/stat.h +++ b/storage/src/vespa/storageapi/message/stat.h diff --git a/storageapi/src/vespa/storageapi/message/state.cpp b/storage/src/vespa/storageapi/message/state.cpp index 23bd766ac2a..23bd766ac2a 100644 --- a/storageapi/src/vespa/storageapi/message/state.cpp +++ b/storage/src/vespa/storageapi/message/state.cpp diff --git a/storageapi/src/vespa/storageapi/message/state.h b/storage/src/vespa/storageapi/message/state.h index aa562c77ef9..aa562c77ef9 100644 --- a/storageapi/src/vespa/storageapi/message/state.h +++ b/storage/src/vespa/storageapi/message/state.h diff --git a/storageapi/src/vespa/storageapi/message/visitor.cpp b/storage/src/vespa/storageapi/message/visitor.cpp index fb3274273ac..fb3274273ac 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.cpp +++ b/storage/src/vespa/storageapi/message/visitor.cpp diff --git a/storageapi/src/vespa/storageapi/message/visitor.h b/storage/src/vespa/storageapi/message/visitor.h index e6835405768..e6835405768 100644 --- a/storageapi/src/vespa/storageapi/message/visitor.h +++ b/storage/src/vespa/storageapi/message/visitor.h diff --git a/storageapi/src/vespa/storageapi/messageapi/.gitignore b/storage/src/vespa/storageapi/messageapi/.gitignore index 6e555686642..6e555686642 100644 --- a/storageapi/src/vespa/storageapi/messageapi/.gitignore +++ b/storage/src/vespa/storageapi/messageapi/.gitignore diff --git a/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt b/storage/src/vespa/storageapi/messageapi/CMakeLists.txt index 6454bb1de7d..6454bb1de7d 100644 --- a/storageapi/src/vespa/storageapi/messageapi/CMakeLists.txt +++ b/storage/src/vespa/storageapi/messageapi/CMakeLists.txt diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp b/storage/src/vespa/storageapi/messageapi/bucketcommand.cpp index c75267f560d..c75267f560d 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.cpp +++ b/storage/src/vespa/storageapi/messageapi/bucketcommand.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h b/storage/src/vespa/storageapi/messageapi/bucketcommand.h index 605653681b5..605653681b5 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketcommand.h +++ b/storage/src/vespa/storageapi/messageapi/bucketcommand.h diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp b/storage/src/vespa/storageapi/messageapi/bucketinfocommand.cpp index 8d5ad85b8aa..8d5ad85b8aa 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.cpp +++ b/storage/src/vespa/storageapi/messageapi/bucketinfocommand.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h b/storage/src/vespa/storageapi/messageapi/bucketinfocommand.h index 0f6627328a9..0f6627328a9 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinfocommand.h +++ b/storage/src/vespa/storageapi/messageapi/bucketinfocommand.h diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp b/storage/src/vespa/storageapi/messageapi/bucketinforeply.cpp index 6eb5f96e888..6eb5f96e888 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.cpp +++ b/storage/src/vespa/storageapi/messageapi/bucketinforeply.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h b/storage/src/vespa/storageapi/messageapi/bucketinforeply.h index 961e63a7ac8..961e63a7ac8 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketinforeply.h +++ b/storage/src/vespa/storageapi/messageapi/bucketinforeply.h diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp b/storage/src/vespa/storageapi/messageapi/bucketreply.cpp index 08b5effbd11..08b5effbd11 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketreply.cpp +++ b/storage/src/vespa/storageapi/messageapi/bucketreply.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/bucketreply.h b/storage/src/vespa/storageapi/messageapi/bucketreply.h index e7ded37c14d..e7ded37c14d 100644 --- a/storageapi/src/vespa/storageapi/messageapi/bucketreply.h +++ b/storage/src/vespa/storageapi/messageapi/bucketreply.h diff --git a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.cpp b/storage/src/vespa/storageapi/messageapi/maintenancecommand.cpp index 91551be0987..91551be0987 100644 --- a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.cpp +++ b/storage/src/vespa/storageapi/messageapi/maintenancecommand.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h b/storage/src/vespa/storageapi/messageapi/maintenancecommand.h index 6bb36d0d32f..6bb36d0d32f 100644 --- a/storageapi/src/vespa/storageapi/messageapi/maintenancecommand.h +++ b/storage/src/vespa/storageapi/messageapi/maintenancecommand.h diff --git a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h b/storage/src/vespa/storageapi/messageapi/messagehandler.h index 9ba8542e9db..9ba8542e9db 100644 --- a/storageapi/src/vespa/storageapi/messageapi/messagehandler.h +++ b/storage/src/vespa/storageapi/messageapi/messagehandler.h diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp b/storage/src/vespa/storageapi/messageapi/returncode.cpp index ef587968515..ef587968515 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.cpp +++ b/storage/src/vespa/storageapi/messageapi/returncode.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/returncode.h b/storage/src/vespa/storageapi/messageapi/returncode.h index f60cbe2b840..f60cbe2b840 100644 --- a/storageapi/src/vespa/storageapi/messageapi/returncode.h +++ b/storage/src/vespa/storageapi/messageapi/returncode.h diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp b/storage/src/vespa/storageapi/messageapi/storagecommand.cpp index 1e797ba4792..1e797ba4792 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagecommand.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagecommand.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h b/storage/src/vespa/storageapi/messageapi/storagecommand.h index 30d59e5fe4b..30d59e5fe4b 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagecommand.h +++ b/storage/src/vespa/storageapi/messageapi/storagecommand.h diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp index db5c86af989..db5c86af989 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h b/storage/src/vespa/storageapi/messageapi/storagemessage.h index 71567192bd9..71567192bd9 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagemessage.h +++ b/storage/src/vespa/storageapi/messageapi/storagemessage.h diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp b/storage/src/vespa/storageapi/messageapi/storagereply.cpp index a6fefe57e08..a6fefe57e08 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.cpp +++ b/storage/src/vespa/storageapi/messageapi/storagereply.cpp diff --git a/storageapi/src/vespa/storageapi/messageapi/storagereply.h b/storage/src/vespa/storageapi/messageapi/storagereply.h index 9128617096d..9128617096d 100644 --- a/storageapi/src/vespa/storageapi/messageapi/storagereply.h +++ b/storage/src/vespa/storageapi/messageapi/storagereply.h diff --git a/storageframework/src/vespa/storageframework/.gitignore b/storage/src/vespa/storageframework/.gitignore index fb3ac588d47..fb3ac588d47 100644 --- a/storageframework/src/vespa/storageframework/.gitignore +++ b/storage/src/vespa/storageframework/.gitignore diff --git a/storageframework/src/tests/clock/.gitignore b/storage/src/vespa/storageframework/defaultimplementation/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/tests/clock/.gitignore +++ b/storage/src/vespa/storageframework/defaultimplementation/.gitignore diff --git a/storageframework/src/tests/thread/.gitignore b/storage/src/vespa/storageframework/defaultimplementation/clock/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/tests/thread/.gitignore +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/.gitignore diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/CMakeLists.txt b/storage/src/vespa/storageframework/defaultimplementation/clock/CMakeLists.txt index 9e22f36e6c3..9e22f36e6c3 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/CMakeLists.txt +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.cpp b/storage/src/vespa/storageframework/defaultimplementation/clock/fakeclock.cpp index d02cecd79bb..d02cecd79bb 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/fakeclock.cpp diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h b/storage/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h index f4cdcf2b8ce..f4cdcf2b8ce 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/fakeclock.h diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.cpp b/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.cpp index 0303481feb5..0303481feb5 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.cpp diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.h b/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.h index a4b80a990c9..a4b80a990c9 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/realclock.h +++ b/storage/src/vespa/storageframework/defaultimplementation/clock/realclock.h diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/.gitignore b/storage/src/vespa/storageframework/defaultimplementation/component/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/.gitignore +++ b/storage/src/vespa/storageframework/defaultimplementation/component/.gitignore diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/CMakeLists.txt b/storage/src/vespa/storageframework/defaultimplementation/component/CMakeLists.txt index 5181d709dc3..5181d709dc3 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/CMakeLists.txt +++ b/storage/src/vespa/storageframework/defaultimplementation/component/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp index 0da02b15a11..0da02b15a11 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.cpp diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h index 6cfae8086fc..6cfae8086fc 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/component/componentregisterimpl.h diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp index 945e264aaea..945e264aaea 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.cpp diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h index bd4afa6c9ad..bd4afa6c9ad 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h +++ b/storage/src/vespa/storageframework/defaultimplementation/component/testcomponentregister.h diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/clock/.gitignore b/storage/src/vespa/storageframework/defaultimplementation/thread/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/clock/.gitignore +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/.gitignore diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/CMakeLists.txt b/storage/src/vespa/storageframework/defaultimplementation/thread/CMakeLists.txt index 950bc527ece..950bc527ece 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/CMakeLists.txt +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp index b8ef8e4610b..b8ef8e4610b 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.cpp diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h index 46a9412bf67..46a9412bf67 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadimpl.h diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp index 7711eddf51c..7711eddf51c 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.cpp diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h index d6053a2f128..d6053a2f128 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h +++ b/storage/src/vespa/storageframework/defaultimplementation/thread/threadpoolimpl.h diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/component/.gitignore b/storage/src/vespa/storageframework/generic/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/component/.gitignore +++ b/storage/src/vespa/storageframework/generic/.gitignore diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/thread/.gitignore b/storage/src/vespa/storageframework/generic/clock/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/defaultimplementation/thread/.gitignore +++ b/storage/src/vespa/storageframework/generic/clock/.gitignore diff --git a/storageframework/src/vespa/storageframework/generic/clock/CMakeLists.txt b/storage/src/vespa/storageframework/generic/clock/CMakeLists.txt index c95860f26fe..c95860f26fe 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/CMakeLists.txt +++ b/storage/src/vespa/storageframework/generic/clock/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/generic/clock/clock.h b/storage/src/vespa/storageframework/generic/clock/clock.h index c9b8f652bfe..c9b8f652bfe 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/clock.h +++ b/storage/src/vespa/storageframework/generic/clock/clock.h diff --git a/storageframework/src/vespa/storageframework/generic/clock/time.cpp b/storage/src/vespa/storageframework/generic/clock/time.cpp index 7bf3fca3835..7bf3fca3835 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/time.cpp +++ b/storage/src/vespa/storageframework/generic/clock/time.cpp diff --git a/storageframework/src/vespa/storageframework/generic/clock/time.h b/storage/src/vespa/storageframework/generic/clock/time.h index 9140ee67332..9140ee67332 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/time.h +++ b/storage/src/vespa/storageframework/generic/clock/time.h diff --git a/storageframework/src/vespa/storageframework/generic/clock/time.hpp b/storage/src/vespa/storageframework/generic/clock/time.hpp index 2cc3fb7be9d..2cc3fb7be9d 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/time.hpp +++ b/storage/src/vespa/storageframework/generic/clock/time.hpp diff --git a/storageframework/src/vespa/storageframework/generic/clock/timer.h b/storage/src/vespa/storageframework/generic/clock/timer.h index 2642a6decc4..2642a6decc4 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/timer.h +++ b/storage/src/vespa/storageframework/generic/clock/timer.h diff --git a/storageframework/src/vespa/storageframework/generic/.gitignore b/storage/src/vespa/storageframework/generic/component/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/generic/.gitignore +++ b/storage/src/vespa/storageframework/generic/component/.gitignore diff --git a/storageframework/src/vespa/storageframework/generic/component/CMakeLists.txt b/storage/src/vespa/storageframework/generic/component/CMakeLists.txt index 69e639b89db..69e639b89db 100644 --- a/storageframework/src/vespa/storageframework/generic/component/CMakeLists.txt +++ b/storage/src/vespa/storageframework/generic/component/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/generic/component/component.cpp b/storage/src/vespa/storageframework/generic/component/component.cpp index f45e1a0b839..f45e1a0b839 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.cpp +++ b/storage/src/vespa/storageframework/generic/component/component.cpp diff --git a/storageframework/src/vespa/storageframework/generic/component/component.h b/storage/src/vespa/storageframework/generic/component/component.h index 09af1d276f3..09af1d276f3 100644 --- a/storageframework/src/vespa/storageframework/generic/component/component.h +++ b/storage/src/vespa/storageframework/generic/component/component.h diff --git a/storageframework/src/vespa/storageframework/generic/component/componentregister.h b/storage/src/vespa/storageframework/generic/component/componentregister.h index da6aa3b51de..da6aa3b51de 100644 --- a/storageframework/src/vespa/storageframework/generic/component/componentregister.h +++ b/storage/src/vespa/storageframework/generic/component/componentregister.h diff --git a/storageframework/src/vespa/storageframework/generic/component/managedcomponent.h b/storage/src/vespa/storageframework/generic/component/managedcomponent.h index 4cd142420cd..4cd142420cd 100644 --- a/storageframework/src/vespa/storageframework/generic/component/managedcomponent.h +++ b/storage/src/vespa/storageframework/generic/component/managedcomponent.h diff --git a/storageframework/src/vespa/storageframework/generic/clock/.gitignore b/storage/src/vespa/storageframework/generic/metric/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/generic/clock/.gitignore +++ b/storage/src/vespa/storageframework/generic/metric/.gitignore diff --git a/storageframework/src/vespa/storageframework/generic/metric/CMakeLists.txt b/storage/src/vespa/storageframework/generic/metric/CMakeLists.txt index 5881159862e..5881159862e 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/CMakeLists.txt +++ b/storage/src/vespa/storageframework/generic/metric/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/generic/metric/metricregistrator.h b/storage/src/vespa/storageframework/generic/metric/metricregistrator.h index 1129bf29e45..1129bf29e45 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/metricregistrator.h +++ b/storage/src/vespa/storageframework/generic/metric/metricregistrator.h diff --git a/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h b/storage/src/vespa/storageframework/generic/metric/metricupdatehook.h index b45713cd0ce..b45713cd0ce 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/metricupdatehook.h +++ b/storage/src/vespa/storageframework/generic/metric/metricupdatehook.h diff --git a/storageframework/src/vespa/storageframework/generic/component/.gitignore b/storage/src/vespa/storageframework/generic/status/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/generic/component/.gitignore +++ b/storage/src/vespa/storageframework/generic/status/.gitignore diff --git a/storageframework/src/vespa/storageframework/generic/status/CMakeLists.txt b/storage/src/vespa/storageframework/generic/status/CMakeLists.txt index a629e632b78..a629e632b78 100644 --- a/storageframework/src/vespa/storageframework/generic/status/CMakeLists.txt +++ b/storage/src/vespa/storageframework/generic/status/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp index 9b7c4919403..9b7c4919403 100644 --- a/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp +++ b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.cpp diff --git a/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.h b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.h index 4ffba20a3fa..4ffba20a3fa 100644 --- a/storageframework/src/vespa/storageframework/generic/status/htmlstatusreporter.h +++ b/storage/src/vespa/storageframework/generic/status/htmlstatusreporter.h diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp b/storage/src/vespa/storageframework/generic/status/httpurlpath.cpp index b55b6dee06e..b55b6dee06e 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.cpp +++ b/storage/src/vespa/storageframework/generic/status/httpurlpath.cpp diff --git a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h b/storage/src/vespa/storageframework/generic/status/httpurlpath.h index 4835365259f..4835365259f 100644 --- a/storageframework/src/vespa/storageframework/generic/status/httpurlpath.h +++ b/storage/src/vespa/storageframework/generic/status/httpurlpath.h diff --git a/storageframework/src/vespa/storageframework/generic/status/statusreporter.cpp b/storage/src/vespa/storageframework/generic/status/statusreporter.cpp index 7314dddde7f..7314dddde7f 100644 --- a/storageframework/src/vespa/storageframework/generic/status/statusreporter.cpp +++ b/storage/src/vespa/storageframework/generic/status/statusreporter.cpp diff --git a/storageframework/src/vespa/storageframework/generic/status/statusreporter.h b/storage/src/vespa/storageframework/generic/status/statusreporter.h index cc48bb841fd..cc48bb841fd 100644 --- a/storageframework/src/vespa/storageframework/generic/status/statusreporter.h +++ b/storage/src/vespa/storageframework/generic/status/statusreporter.h diff --git a/storageframework/src/vespa/storageframework/generic/status/statusreportermap.h b/storage/src/vespa/storageframework/generic/status/statusreportermap.h index 8d72215df49..8d72215df49 100644 --- a/storageframework/src/vespa/storageframework/generic/status/statusreportermap.h +++ b/storage/src/vespa/storageframework/generic/status/statusreportermap.h diff --git a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.cpp b/storage/src/vespa/storageframework/generic/status/xmlstatusreporter.cpp index 258a3ea53de..258a3ea53de 100644 --- a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.cpp +++ b/storage/src/vespa/storageframework/generic/status/xmlstatusreporter.cpp diff --git a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h b/storage/src/vespa/storageframework/generic/status/xmlstatusreporter.h index eff5a44148e..eff5a44148e 100644 --- a/storageframework/src/vespa/storageframework/generic/status/xmlstatusreporter.h +++ b/storage/src/vespa/storageframework/generic/status/xmlstatusreporter.h diff --git a/storageframework/src/vespa/storageframework/generic/metric/.gitignore b/storage/src/vespa/storageframework/generic/thread/.gitignore index 7e7c0fe7fae..7e7c0fe7fae 100644 --- a/storageframework/src/vespa/storageframework/generic/metric/.gitignore +++ b/storage/src/vespa/storageframework/generic/thread/.gitignore diff --git a/storageframework/src/vespa/storageframework/generic/thread/CMakeLists.txt b/storage/src/vespa/storageframework/generic/thread/CMakeLists.txt index a392725b771..a392725b771 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/CMakeLists.txt +++ b/storage/src/vespa/storageframework/generic/thread/CMakeLists.txt diff --git a/storageframework/src/vespa/storageframework/generic/thread/runnable.h b/storage/src/vespa/storageframework/generic/thread/runnable.h index b1fd2eae237..b1fd2eae237 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/runnable.h +++ b/storage/src/vespa/storageframework/generic/thread/runnable.h diff --git a/storageframework/src/vespa/storageframework/generic/thread/taskthread.h b/storage/src/vespa/storageframework/generic/thread/taskthread.h index 49f61fa0f13..49f61fa0f13 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/taskthread.h +++ b/storage/src/vespa/storageframework/generic/thread/taskthread.h diff --git a/storageframework/src/vespa/storageframework/generic/thread/thread.cpp b/storage/src/vespa/storageframework/generic/thread/thread.cpp index 388ac93a9b5..388ac93a9b5 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/thread.cpp +++ b/storage/src/vespa/storageframework/generic/thread/thread.cpp diff --git a/storageframework/src/vespa/storageframework/generic/thread/thread.h b/storage/src/vespa/storageframework/generic/thread/thread.h index c17638a0d42..c17638a0d42 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/thread.h +++ b/storage/src/vespa/storageframework/generic/thread/thread.h diff --git a/storageframework/src/vespa/storageframework/generic/thread/threadpool.cpp b/storage/src/vespa/storageframework/generic/thread/threadpool.cpp index 480e42c91ef..480e42c91ef 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/threadpool.cpp +++ b/storage/src/vespa/storageframework/generic/thread/threadpool.cpp diff --git a/storageframework/src/vespa/storageframework/generic/thread/threadpool.h b/storage/src/vespa/storageframework/generic/thread/threadpool.h index 7607932e079..7607932e079 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/threadpool.h +++ b/storage/src/vespa/storageframework/generic/thread/threadpool.h diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp b/storage/src/vespa/storageframework/generic/thread/tickingthread.cpp index 1a9cb459f28..1a9cb459f28 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.cpp +++ b/storage/src/vespa/storageframework/generic/thread/tickingthread.cpp diff --git a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h b/storage/src/vespa/storageframework/generic/thread/tickingthread.h index 9ddc47c8f3a..9ddc47c8f3a 100644 --- a/storageframework/src/vespa/storageframework/generic/thread/tickingthread.h +++ b/storage/src/vespa/storageframework/generic/thread/tickingthread.h diff --git a/storageframework/src/vespa/storageframework/storageframework.h b/storage/src/vespa/storageframework/storageframework.h index 12d4f504d30..12d4f504d30 100644 --- a/storageframework/src/vespa/storageframework/storageframework.h +++ b/storage/src/vespa/storageframework/storageframework.h diff --git a/storageapi/.gitignore b/storageapi/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/storageapi/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/storageapi/CMakeLists.txt b/storageapi/CMakeLists.txt deleted file mode 100644 index d8e00cc35bf..00000000000 --- a/storageapi/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - fnet - vespalog - vespalib - document - messagebus - vdslib - documentapi - - LIBS - src/vespa/storageapi - src/vespa/storageapi/app - src/vespa/storageapi/buckets - src/vespa/storageapi/mbusprot - src/vespa/storageapi/message - src/vespa/storageapi/messageapi - - TESTS - src/tests - src/tests/buckets - src/tests/mbusprot - src/tests/messageapi -) diff --git a/storageapi/OWNERS b/storageapi/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/storageapi/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/storageapi/README b/storageapi/README deleted file mode 100644 index f6332c9f8f0..00000000000 --- a/storageapi/README +++ /dev/null @@ -1 +0,0 @@ -This is the API needed to access the Vespa Storage System. diff --git a/storageapi/src/.gitignore b/storageapi/src/.gitignore deleted file mode 100644 index a735fdb4395..00000000000 --- a/storageapi/src/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -.cvsignore -Makefile.ini -config_command.sh -doc -project.dsw diff --git a/storageapi/src/tests/messageapi/.gitignore b/storageapi/src/tests/messageapi/.gitignore deleted file mode 100644 index 1d859456ef9..00000000000 --- a/storageapi/src/tests/messageapi/.gitignore +++ /dev/null @@ -1,9 +0,0 @@ -*.So -*.core -*.so -.*.swp -.config.log -.depend -Makefile -test.vlog -testrunner diff --git a/storageapi/src/vespa/storageapi/CMakeLists.txt b/storageapi/src/vespa/storageapi/CMakeLists.txt deleted file mode 100644 index 7018bbc1b55..00000000000 --- a/storageapi/src/vespa/storageapi/CMakeLists.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -vespa_add_library(storageapi - SOURCES - $<TARGET_OBJECTS:storageapi_message> - $<TARGET_OBJECTS:storageapi_buckets> - $<TARGET_OBJECTS:storageapi_messageapi> - $<TARGET_OBJECTS:storageapi_mbusprot> - INSTALL lib64 - DEPENDS -) - -vespa_add_target_package_dependency(storageapi Protobuf) - diff --git a/storageframework/.gitignore b/storageframework/.gitignore deleted file mode 100644 index f3c7a7c5da6..00000000000 --- a/storageframework/.gitignore +++ /dev/null @@ -1 +0,0 @@ -Makefile diff --git a/storageframework/CMakeLists.txt b/storageframework/CMakeLists.txt deleted file mode 100644 index b6e3855e96d..00000000000 --- a/storageframework/CMakeLists.txt +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - vespalog - vespalib - metrics - - LIBS - src/vespa/storageframework - src/vespa/storageframework/defaultimplementation - src/vespa/storageframework/defaultimplementation/clock - src/vespa/storageframework/defaultimplementation/component - src/vespa/storageframework/defaultimplementation/thread - src/vespa/storageframework/generic - src/vespa/storageframework/generic/clock - src/vespa/storageframework/generic/component - src/vespa/storageframework/generic/metric - src/vespa/storageframework/generic/status - src/vespa/storageframework/generic/thread - - TEST_EXTERNAL_DEPENDS - vdstestlib - - TESTS - src/tests - src/tests/clock - src/tests/thread -) diff --git a/storageframework/OWNERS b/storageframework/OWNERS deleted file mode 100644 index dbcff24b338..00000000000 --- a/storageframework/OWNERS +++ /dev/null @@ -1 +0,0 @@ -vekterli diff --git a/storageframework/README b/storageframework/README deleted file mode 100644 index b24aedd74f3..00000000000 --- a/storageframework/README +++ /dev/null @@ -1,10 +0,0 @@ -This module contains a generic application framework for a C++ server -process. It provides functionality such as metrics, status pages, memory -management and deadlock detection. - -This used to be coded into the storage module itself, but we want the generic -stuff to be available to others. The driver for moving is that storage now -splits its own memfile persistence layer out of storage, and we want both parts -to have the same utilities available. - -This module should hopefully not contain very storage specific stuff. diff --git a/storageframework/src/.gitignore b/storageframework/src/.gitignore deleted file mode 100644 index 2e8e6fd906a..00000000000 --- a/storageframework/src/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/Makefile.ini -/config_command.sh -/project.dsw diff --git a/storageframework/src/vespa/storageframework/CMakeLists.txt b/storageframework/src/vespa/storageframework/CMakeLists.txt deleted file mode 100644 index fcc858ee1eb..00000000000 --- a/storageframework/src/vespa/storageframework/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageframework INTERFACE - SOURCES - INSTALL lib64 - DEPENDS - storageframework_defaultimplementation - storageframework_generic -) diff --git a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt b/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt deleted file mode 100644 index b10bb014c0c..00000000000 --- a/storageframework/src/vespa/storageframework/defaultimplementation/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageframework_defaultimplementation - SOURCES - $<TARGET_OBJECTS:storageframework_clockimpl> - $<TARGET_OBJECTS:storageframework_componentimpl> - $<TARGET_OBJECTS:storageframework_threadimpl> - INSTALL lib64 - DEPENDS - storageframework_generic -) diff --git a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt b/storageframework/src/vespa/storageframework/generic/CMakeLists.txt deleted file mode 100644 index 2507fc8cf41..00000000000 --- a/storageframework/src/vespa/storageframework/generic/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(storageframework_generic - SOURCES - $<TARGET_OBJECTS:storageframework_component> - $<TARGET_OBJECTS:storageframework_status> - $<TARGET_OBJECTS:storageframework_thread> - $<TARGET_OBJECTS:storageframework_clock> - INSTALL lib64 - DEPENDS -) diff --git a/storageframework/src/vespa/storageframework/generic/status/.gitignore b/storageframework/src/vespa/storageframework/generic/status/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/storageframework/src/vespa/storageframework/generic/status/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/storageframework/src/vespa/storageframework/generic/thread/.gitignore b/storageframework/src/vespa/storageframework/generic/thread/.gitignore deleted file mode 100644 index 7e7c0fe7fae..00000000000 --- a/storageframework/src/vespa/storageframework/generic/thread/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/.depend -/Makefile diff --git a/storageserver/CMakeLists.txt b/storageserver/CMakeLists.txt index 628dc9ece3e..ee3335c4921 100644 --- a/storageserver/CMakeLists.txt +++ b/storageserver/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_define_module( DEPENDS fastos storage - streamingvisitors_searchvisitor + streamingvisitors APPS src/apps/storaged diff --git a/storageserver/src/apps/storaged/CMakeLists.txt b/storageserver/src/apps/storaged/CMakeLists.txt index 3fb02b5d1b9..10f38cf5269 100644 --- a/storageserver/src/apps/storaged/CMakeLists.txt +++ b/storageserver/src/apps/storaged/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_executable(storageserver_storaged_app INSTALL sbin DEPENDS storageserver_storageapp - searchlib_searchlib_uca ) install(CODE "execute_process(COMMAND ln -snf vespa-storaged-bin \$ENV{DESTDIR}/\${CMAKE_INSTALL_PREFIX}/sbin/vespa-distributord-bin)") diff --git a/storageserver/src/vespa/storageserver/app/CMakeLists.txt b/storageserver/src/vespa/storageserver/app/CMakeLists.txt index 2be219c508c..67bafb2256f 100644 --- a/storageserver/src/vespa/storageserver/app/CMakeLists.txt +++ b/storageserver/src/vespa/storageserver/app/CMakeLists.txt @@ -6,5 +6,5 @@ vespa_add_library(storageserver_storageapp STATIC servicelayerprocess.cpp dummyservicelayerprocess.cpp DEPENDS - storage_storageserver + storage ) diff --git a/streamingvisitors/CMakeLists.txt b/streamingvisitors/CMakeLists.txt index 5f5e16fc6c3..0e7789a21b9 100644 --- a/streamingvisitors/CMakeLists.txt +++ b/streamingvisitors/CMakeLists.txt @@ -2,22 +2,30 @@ vespa_define_module( DEPENDS fastos - fastlib_fast vespalog storage - storageapi config_cloudconfig - document vespalib + document vdslib - vsm + searchlib + searchsummary LIBS src/vespa/searchvisitor + src/vespa/vsm/common + src/vespa/vsm/config + src/vespa/vsm/searcher + src/vespa/vsm/vsm TESTS src/tests/hitcollector src/tests/matching_elements_filler src/tests/querywrapper src/tests/searchvisitor + src/tests/charbuffer + src/tests/docsum + src/tests/document + src/tests/searcher + src/tests/textutil ) diff --git a/vsm/pom.xml b/streamingvisitors/pom.xml index 2cc777be593..2cc777be593 100644 --- a/vsm/pom.xml +++ b/streamingvisitors/pom.xml diff --git a/vsm/src/tests/charbuffer/.gitignore b/streamingvisitors/src/tests/charbuffer/.gitignore index 2c980038fb5..2c980038fb5 100644 --- a/vsm/src/tests/charbuffer/.gitignore +++ b/streamingvisitors/src/tests/charbuffer/.gitignore diff --git a/vsm/src/tests/charbuffer/CMakeLists.txt b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt index 43eb6a8114f..5d0c0068d37 100644 --- a/vsm/src/tests/charbuffer/CMakeLists.txt +++ b/streamingvisitors/src/tests/charbuffer/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vsm_charbuffer_test_app TEST SOURCES charbuffer.cpp DEPENDS - vsm + streamingvisitors ) vespa_add_test(NAME vsm_charbuffer_test_app COMMAND vsm_charbuffer_test_app) diff --git a/vsm/src/tests/charbuffer/charbuffer.cpp b/streamingvisitors/src/tests/charbuffer/charbuffer.cpp index 736d35459cb..736d35459cb 100644 --- a/vsm/src/tests/charbuffer/charbuffer.cpp +++ b/streamingvisitors/src/tests/charbuffer/charbuffer.cpp diff --git a/vsm/src/tests/config/mail.cfg b/streamingvisitors/src/tests/config/mail.cfg index ce830beac23..ce830beac23 100644 --- a/vsm/src/tests/config/mail.cfg +++ b/streamingvisitors/src/tests/config/mail.cfg diff --git a/vsm/src/tests/config/vsm.cfg b/streamingvisitors/src/tests/config/vsm.cfg index dc50447f623..dc50447f623 100644 --- a/vsm/src/tests/config/vsm.cfg +++ b/streamingvisitors/src/tests/config/vsm.cfg diff --git a/vsm/src/tests/config/vsmfields.cfg b/streamingvisitors/src/tests/config/vsmfields.cfg index 30f1c8ed8b1..30f1c8ed8b1 100644 --- a/vsm/src/tests/config/vsmfields.cfg +++ b/streamingvisitors/src/tests/config/vsmfields.cfg diff --git a/vsm/src/tests/docsum/.gitignore b/streamingvisitors/src/tests/docsum/.gitignore index 9a697a94de8..9a697a94de8 100644 --- a/vsm/src/tests/docsum/.gitignore +++ b/streamingvisitors/src/tests/docsum/.gitignore diff --git a/vsm/src/tests/docsum/CMakeLists.txt b/streamingvisitors/src/tests/docsum/CMakeLists.txt index aec2637b3ad..87c46409053 100644 --- a/vsm/src/tests/docsum/CMakeLists.txt +++ b/streamingvisitors/src/tests/docsum/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vsm_docsum_test_app TEST SOURCES docsum.cpp DEPENDS - vsm + streamingvisitors ) vespa_add_test(NAME vsm_docsum_test_app COMMAND vsm_docsum_test_app) diff --git a/vsm/src/tests/docsum/docsum.cpp b/streamingvisitors/src/tests/docsum/docsum.cpp index 475489d2f5a..475489d2f5a 100644 --- a/vsm/src/tests/docsum/docsum.cpp +++ b/streamingvisitors/src/tests/docsum/docsum.cpp diff --git a/vsm/src/tests/document/.gitignore b/streamingvisitors/src/tests/document/.gitignore index d47781eff63..d47781eff63 100644 --- a/vsm/src/tests/document/.gitignore +++ b/streamingvisitors/src/tests/document/.gitignore diff --git a/vsm/src/tests/document/CMakeLists.txt b/streamingvisitors/src/tests/document/CMakeLists.txt index 819d300280f..5ea12dc5e2d 100644 --- a/vsm/src/tests/document/CMakeLists.txt +++ b/streamingvisitors/src/tests/document/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vsm_document_test_app TEST SOURCES document.cpp DEPENDS - vsm + streamingvisitors ) vespa_add_test(NAME vsm_document_test_app COMMAND vsm_document_test_app) diff --git a/vsm/src/tests/document/document.cpp b/streamingvisitors/src/tests/document/document.cpp index 1e97d232a64..1e97d232a64 100644 --- a/vsm/src/tests/document/document.cpp +++ b/streamingvisitors/src/tests/document/document.cpp diff --git a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt index f25ab348265..dbec820a462 100644 --- a/streamingvisitors/src/tests/hitcollector/CMakeLists.txt +++ b/streamingvisitors/src/tests/hitcollector/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_hitcollector_test_app TEST SOURCES hitcollector_test.cpp DEPENDS - streamingvisitors_searchvisitor + streamingvisitors ) vespa_add_test(NAME streamingvisitors_hitcollector_test_app COMMAND streamingvisitors_hitcollector_test_app) diff --git a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt index ef93d551912..5cc2977b3c3 100644 --- a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt +++ b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_executable(streamingvisitors_matching_elements_filler_test_app TEST SOURCES matching_elements_filler_test.cpp DEPENDS - streamingvisitors_searchvisitor + streamingvisitors GTest::GTest ) vespa_add_test(NAME streamingvisitors_matching_elements_filler_test_app COMMAND streamingvisitors_matching_elements_filler_test_app) diff --git a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt index 7cae60e6a11..e0131d0c6cc 100644 --- a/streamingvisitors/src/tests/querywrapper/CMakeLists.txt +++ b/streamingvisitors/src/tests/querywrapper/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_querywrapper_test_app TEST SOURCES querywrapper_test.cpp DEPENDS - streamingvisitors_searchvisitor + streamingvisitors ) vespa_add_test(NAME streamingvisitors_querywrapper_test_app COMMAND streamingvisitors_querywrapper_test_app) diff --git a/vsm/src/tests/searcher/.gitignore b/streamingvisitors/src/tests/searcher/.gitignore index 52a56dff405..52a56dff405 100644 --- a/vsm/src/tests/searcher/.gitignore +++ b/streamingvisitors/src/tests/searcher/.gitignore diff --git a/vsm/src/tests/searcher/CMakeLists.txt b/streamingvisitors/src/tests/searcher/CMakeLists.txt index 9e0bf0391ef..2277f5ef55f 100644 --- a/vsm/src/tests/searcher/CMakeLists.txt +++ b/streamingvisitors/src/tests/searcher/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vsm_searcher_test_app TEST SOURCES searcher_test.cpp DEPENDS - vsm + streamingvisitors ) vespa_add_test(NAME vsm_searcher_test_app COMMAND vsm_searcher_test_app) diff --git a/vsm/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp index 34fa66eaa90..34fa66eaa90 100644 --- a/vsm/src/tests/searcher/searcher_test.cpp +++ b/streamingvisitors/src/tests/searcher/searcher_test.cpp diff --git a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt index fdbd60ce30a..01b625b6b3b 100644 --- a/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt +++ b/streamingvisitors/src/tests/searchvisitor/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(streamingvisitors_searchvisitor_test_app TEST SOURCES searchvisitor_test.cpp DEPENDS - streamingvisitors_searchvisitor + streamingvisitors ) vespa_add_test(NAME streamingvisitors_searchvisitor_test_app COMMAND streamingvisitors_searchvisitor_test_app) diff --git a/vsm/src/tests/textutil/.gitignore b/streamingvisitors/src/tests/textutil/.gitignore index 1103f79800a..1103f79800a 100644 --- a/vsm/src/tests/textutil/.gitignore +++ b/streamingvisitors/src/tests/textutil/.gitignore diff --git a/vsm/src/tests/textutil/CMakeLists.txt b/streamingvisitors/src/tests/textutil/CMakeLists.txt index 83d4d8b00d6..59817d01137 100644 --- a/vsm/src/tests/textutil/CMakeLists.txt +++ b/streamingvisitors/src/tests/textutil/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(vsm_textutil_test_app TEST SOURCES textutil.cpp DEPENDS - vsm + streamingvisitors ) vespa_add_test(NAME vsm_textutil_test_app COMMAND vsm_textutil_test_app) diff --git a/vsm/src/tests/textutil/textutil.cpp b/streamingvisitors/src/tests/textutil/textutil.cpp index 2a1390eaa01..2a1390eaa01 100644 --- a/vsm/src/tests/textutil/textutil.cpp +++ b/streamingvisitors/src/tests/textutil/textutil.cpp diff --git a/staging_vespalib/src/tests/health_server/.gitignore b/streamingvisitors/src/tests/utilapps/.gitignore index e69de29bb2d..e69de29bb2d 100644 --- a/staging_vespalib/src/tests/health_server/.gitignore +++ b/streamingvisitors/src/tests/utilapps/.gitignore diff --git a/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt b/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt index e8f85fc987e..8a3bd92f5a8 100644 --- a/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt +++ b/streamingvisitors/src/vespa/searchvisitor/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(streamingvisitors_searchvisitor +vespa_add_library(streamingvisitors SOURCES hitcollector.cpp indexenvironment.cpp @@ -11,7 +11,10 @@ vespa_add_library(streamingvisitors_searchvisitor rankprocessor.cpp searchenvironment.cpp searchvisitor.cpp + $<TARGET_OBJECTS:vsm_vconfig> + $<TARGET_OBJECTS:vsm_vsmbase> + $<TARGET_OBJECTS:vsm_vsmcommon> + $<TARGET_OBJECTS:vsm_vsmsearcher> INSTALL lib64 DEPENDS - searchlib_searchlib_uca ) diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index 73baa1de45f..e31341c466c 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -95,6 +95,7 @@ RankProcessor::initQueryEnvironment() terms[i].getTerm()->index().c_str(), terms[i].getTerm()->getTerm()); } } + _rankSetup.prepareSharedState(_queryEnv, _queryEnv.getObjectStore()); _match_data = _mdLayout.createMatchData(); } diff --git a/streamingvisitors/src/vespa/snippetvisitor/.gitignore b/streamingvisitors/src/vespa/snippetvisitor/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/streamingvisitors/src/vespa/snippetvisitor/.gitignore +++ /dev/null diff --git a/vsm/src/vespa/vsm/.gitignore b/streamingvisitors/src/vespa/vsm/.gitignore index 4c5f5d9ef7a..4c5f5d9ef7a 100644 --- a/vsm/src/vespa/vsm/.gitignore +++ b/streamingvisitors/src/vespa/vsm/.gitignore diff --git a/vsm/src/vespa/vsm/common/.gitignore b/streamingvisitors/src/vespa/vsm/common/.gitignore index 95bc02923a9..95bc02923a9 100644 --- a/vsm/src/vespa/vsm/common/.gitignore +++ b/streamingvisitors/src/vespa/vsm/common/.gitignore diff --git a/vsm/src/vespa/vsm/common/CMakeLists.txt b/streamingvisitors/src/vespa/vsm/common/CMakeLists.txt index 4570a9b581e..4570a9b581e 100644 --- a/vsm/src/vespa/vsm/common/CMakeLists.txt +++ b/streamingvisitors/src/vespa/vsm/common/CMakeLists.txt diff --git a/vsm/src/vespa/vsm/common/charbuffer.cpp b/streamingvisitors/src/vespa/vsm/common/charbuffer.cpp index b8fbb5c8846..b8fbb5c8846 100644 --- a/vsm/src/vespa/vsm/common/charbuffer.cpp +++ b/streamingvisitors/src/vespa/vsm/common/charbuffer.cpp diff --git a/vsm/src/vespa/vsm/common/charbuffer.h b/streamingvisitors/src/vespa/vsm/common/charbuffer.h index 08618a9b973..08618a9b973 100644 --- a/vsm/src/vespa/vsm/common/charbuffer.h +++ b/streamingvisitors/src/vespa/vsm/common/charbuffer.h diff --git a/vsm/src/vespa/vsm/common/docsum.h b/streamingvisitors/src/vespa/vsm/common/docsum.h index 49b84cb0783..49b84cb0783 100644 --- a/vsm/src/vespa/vsm/common/docsum.h +++ b/streamingvisitors/src/vespa/vsm/common/docsum.h diff --git a/vsm/src/vespa/vsm/common/document.cpp b/streamingvisitors/src/vespa/vsm/common/document.cpp index a345c82ce2d..a345c82ce2d 100644 --- a/vsm/src/vespa/vsm/common/document.cpp +++ b/streamingvisitors/src/vespa/vsm/common/document.cpp diff --git a/vsm/src/vespa/vsm/common/document.h b/streamingvisitors/src/vespa/vsm/common/document.h index 8c11d27072b..8c11d27072b 100644 --- a/vsm/src/vespa/vsm/common/document.h +++ b/streamingvisitors/src/vespa/vsm/common/document.h diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.cpp b/streamingvisitors/src/vespa/vsm/common/documenttypemapping.cpp index 7886c44b2e0..7886c44b2e0 100644 --- a/vsm/src/vespa/vsm/common/documenttypemapping.cpp +++ b/streamingvisitors/src/vespa/vsm/common/documenttypemapping.cpp diff --git a/vsm/src/vespa/vsm/common/documenttypemapping.h b/streamingvisitors/src/vespa/vsm/common/documenttypemapping.h index 607b40cec47..607b40cec47 100644 --- a/vsm/src/vespa/vsm/common/documenttypemapping.h +++ b/streamingvisitors/src/vespa/vsm/common/documenttypemapping.h diff --git a/vsm/src/vespa/vsm/common/fieldmodifier.cpp b/streamingvisitors/src/vespa/vsm/common/fieldmodifier.cpp index b39afd83b5a..b39afd83b5a 100644 --- a/vsm/src/vespa/vsm/common/fieldmodifier.cpp +++ b/streamingvisitors/src/vespa/vsm/common/fieldmodifier.cpp diff --git a/vsm/src/vespa/vsm/common/fieldmodifier.h b/streamingvisitors/src/vespa/vsm/common/fieldmodifier.h index 60e480fa237..60e480fa237 100644 --- a/vsm/src/vespa/vsm/common/fieldmodifier.h +++ b/streamingvisitors/src/vespa/vsm/common/fieldmodifier.h diff --git a/vsm/src/vespa/vsm/common/storagedocument.cpp b/streamingvisitors/src/vespa/vsm/common/storagedocument.cpp index a0d666268f5..a0d666268f5 100644 --- a/vsm/src/vespa/vsm/common/storagedocument.cpp +++ b/streamingvisitors/src/vespa/vsm/common/storagedocument.cpp diff --git a/vsm/src/vespa/vsm/common/storagedocument.h b/streamingvisitors/src/vespa/vsm/common/storagedocument.h index a7f21cb052f..a7f21cb052f 100644 --- a/vsm/src/vespa/vsm/common/storagedocument.h +++ b/streamingvisitors/src/vespa/vsm/common/storagedocument.h diff --git a/vsm/src/vespa/vsm/config/.gitignore b/streamingvisitors/src/vespa/vsm/config/.gitignore index d58390943e2..d58390943e2 100644 --- a/vsm/src/vespa/vsm/config/.gitignore +++ b/streamingvisitors/src/vespa/vsm/config/.gitignore diff --git a/vsm/src/vespa/vsm/config/CMakeLists.txt b/streamingvisitors/src/vespa/vsm/config/CMakeLists.txt index fea0bafe6b2..fea0bafe6b2 100644 --- a/vsm/src/vespa/vsm/config/CMakeLists.txt +++ b/streamingvisitors/src/vespa/vsm/config/CMakeLists.txt diff --git a/vsm/src/vespa/vsm/config/vsm-cfif.h b/streamingvisitors/src/vespa/vsm/config/vsm-cfif.h index 22033aee232..22033aee232 100644 --- a/vsm/src/vespa/vsm/config/vsm-cfif.h +++ b/streamingvisitors/src/vespa/vsm/config/vsm-cfif.h diff --git a/vsm/src/vespa/vsm/config/vsm.def b/streamingvisitors/src/vespa/vsm/config/vsm.def index 1971f9e9574..1971f9e9574 100644 --- a/vsm/src/vespa/vsm/config/vsm.def +++ b/streamingvisitors/src/vespa/vsm/config/vsm.def diff --git a/vsm/src/vespa/vsm/config/vsmfields.def b/streamingvisitors/src/vespa/vsm/config/vsmfields.def index 5e943c9274d..5e943c9274d 100644 --- a/vsm/src/vespa/vsm/config/vsmfields.def +++ b/streamingvisitors/src/vespa/vsm/config/vsmfields.def diff --git a/vsm/src/vespa/vsm/config/vsmsummary.def b/streamingvisitors/src/vespa/vsm/config/vsmsummary.def index 5eb96624826..5eb96624826 100644 --- a/vsm/src/vespa/vsm/config/vsmsummary.def +++ b/streamingvisitors/src/vespa/vsm/config/vsmsummary.def diff --git a/vsm/src/vespa/vsm/searcher/.gitignore b/streamingvisitors/src/vespa/vsm/searcher/.gitignore index 95bc02923a9..95bc02923a9 100644 --- a/vsm/src/vespa/vsm/searcher/.gitignore +++ b/streamingvisitors/src/vespa/vsm/searcher/.gitignore diff --git a/vsm/src/vespa/vsm/searcher/CMakeLists.txt b/streamingvisitors/src/vespa/vsm/searcher/CMakeLists.txt index 0a2a9ec21d2..0a2a9ec21d2 100644 --- a/vsm/src/vespa/vsm/searcher/CMakeLists.txt +++ b/streamingvisitors/src/vespa/vsm/searcher/CMakeLists.txt diff --git a/vsm/src/vespa/vsm/searcher/boolfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.cpp index 8c9b556e593..8c9b556e593 100644 --- a/vsm/src/vespa/vsm/searcher/boolfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/boolfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.h index f6afef9e507..f6afef9e507 100644 --- a/vsm/src/vespa/vsm/searcher/boolfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp index e69999b160e..e69999b160e 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/fieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h index 5c2ef8fec28..5c2ef8fec28 100644 --- a/vsm/src/vespa/vsm/searcher/fieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/floatfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp index 02d8bd8c12a..02d8bd8c12a 100644 --- a/vsm/src/vespa/vsm/searcher/floatfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/floatfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.h index 98018fbf4a3..98018fbf4a3 100644 --- a/vsm/src/vespa/vsm/searcher/floatfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/fold.cpp b/streamingvisitors/src/vespa/vsm/searcher/fold.cpp index bd2392d3ad6..bd2392d3ad6 100644 --- a/vsm/src/vespa/vsm/searcher/fold.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/fold.cpp diff --git a/vsm/src/vespa/vsm/searcher/fold.h b/streamingvisitors/src/vespa/vsm/searcher/fold.h index 578b883484f..578b883484f 100644 --- a/vsm/src/vespa/vsm/searcher/fold.h +++ b/streamingvisitors/src/vespa/vsm/searcher/fold.h diff --git a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp index fc5d77de419..fc5d77de419 100644 --- a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/futf8strchrfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/futf8strchrfieldsearcher.h index 900ab4c9120..900ab4c9120 100644 --- a/vsm/src/vespa/vsm/searcher/futf8strchrfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/futf8strchrfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp index db93bda7778..db93bda7778 100644 --- a/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.h b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.h index ef1c5b5a1c4..ef1c5b5a1c4 100644 --- a/vsm/src/vespa/vsm/searcher/geo_pos_field_searcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.h diff --git a/vsm/src/vespa/vsm/searcher/intfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp index 8cfb8e6df14..8cfb8e6df14 100644 --- a/vsm/src/vespa/vsm/searcher/intfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/intfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.h index a2b17a87f4b..a2b17a87f4b 100644 --- a/vsm/src/vespa/vsm/searcher/intfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp index 1c4ff78ff4a..1c4ff78ff4a 100644 --- a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.h index 0155c79cddf..0155c79cddf 100644 --- a/vsm/src/vespa/vsm/searcher/strchrfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp index 977602a691c..977602a691c 100644 --- a/vsm/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.h index 744974a6cf6..744974a6cf6 100644 --- a/vsm/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8exactstringfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp index 9aef99f9fa1..9aef99f9fa1 100644 --- a/vsm/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h index 63931af0036..63931af0036 100644 --- a/vsm/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp index 0d93009655c..0d93009655c 100644 --- a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.h index 1687a1a18c0..1687a1a18c0 100644 --- a/vsm/src/vespa/vsm/searcher/utf8strchrfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8strchrfieldsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp index 148cdf2c0c3..148cdf2c0c3 100644 --- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h index f540a7ac457..f540a7ac457 100644 --- a/vsm/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.cpp index fd327d3a3df..fd327d3a3df 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.h index 1c463c28847..1c463c28847 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsearcher.h diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp index be02a58cfda..be02a58cfda 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.h b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.h index 0127a7f2827..0127a7f2827 100644 --- a/vsm/src/vespa/vsm/searcher/utf8substringsnippetmodifier.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8substringsnippetmodifier.h diff --git a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp index 3495d46b85b..3495d46b85b 100644 --- a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.cpp diff --git a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.h index 0640ac22da5..0640ac22da5 100644 --- a/vsm/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8suffixstringfieldsearcher.h diff --git a/vsm/src/vespa/vsm/vsm/.gitignore b/streamingvisitors/src/vespa/vsm/vsm/.gitignore index 95bc02923a9..95bc02923a9 100644 --- a/vsm/src/vespa/vsm/vsm/.gitignore +++ b/streamingvisitors/src/vespa/vsm/vsm/.gitignore diff --git a/vsm/src/vespa/vsm/vsm/CMakeLists.txt b/streamingvisitors/src/vespa/vsm/vsm/CMakeLists.txt index adc00b341a3..adc00b341a3 100644 --- a/vsm/src/vespa/vsm/vsm/CMakeLists.txt +++ b/streamingvisitors/src/vespa/vsm/vsm/CMakeLists.txt diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp index 656e9eed132..656e9eed132 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.cpp diff --git a/vsm/src/vespa/vsm/vsm/docsumconfig.h b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h index 11010c04e90..11010c04e90 100644 --- a/vsm/src/vespa/vsm/vsm/docsumconfig.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumconfig.h diff --git a/vsm/src/vespa/vsm/vsm/docsumfieldspec.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfieldspec.cpp index 936aaaa2091..936aaaa2091 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfieldspec.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfieldspec.cpp diff --git a/vsm/src/vespa/vsm/vsm/docsumfieldspec.h b/streamingvisitors/src/vespa/vsm/vsm/docsumfieldspec.h index db6ee9fa223..db6ee9fa223 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfieldspec.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfieldspec.h diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp index 70759feb41c..70759feb41c 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.cpp diff --git a/vsm/src/vespa/vsm/vsm/docsumfilter.h b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h index e6f7ae3e6fe..e6f7ae3e6fe 100644 --- a/vsm/src/vespa/vsm/vsm/docsumfilter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/docsumfilter.h diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp index 7043e63ec87..7043e63ec87 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.cpp diff --git a/vsm/src/vespa/vsm/vsm/fieldsearchspec.h b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h index 7b78a8634e0..7b78a8634e0 100644 --- a/vsm/src/vespa/vsm/vsm/fieldsearchspec.h +++ b/streamingvisitors/src/vespa/vsm/vsm/fieldsearchspec.h diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp b/streamingvisitors/src/vespa/vsm/vsm/flattendocsumwriter.cpp index 06b652d85e6..06b652d85e6 100644 --- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/flattendocsumwriter.cpp diff --git a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h b/streamingvisitors/src/vespa/vsm/vsm/flattendocsumwriter.h index 47c6f1e75d0..47c6f1e75d0 100644 --- a/vsm/src/vespa/vsm/vsm/flattendocsumwriter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/flattendocsumwriter.h diff --git a/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h b/streamingvisitors/src/vespa/vsm/vsm/i_matching_elements_filler.h index a35cea40cec..a35cea40cec 100644 --- a/vsm/src/vespa/vsm/vsm/i_matching_elements_filler.h +++ b/streamingvisitors/src/vespa/vsm/vsm/i_matching_elements_filler.h diff --git a/vsm/src/vespa/vsm/vsm/slimefieldwriter.cpp b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.cpp index 5bc5798fb9d..5bc5798fb9d 100644 --- a/vsm/src/vespa/vsm/vsm/slimefieldwriter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.cpp diff --git a/vsm/src/vespa/vsm/vsm/slimefieldwriter.h b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h index b5adac8985f..b5adac8985f 100644 --- a/vsm/src/vespa/vsm/vsm/slimefieldwriter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/slimefieldwriter.h diff --git a/vsm/src/vespa/vsm/vsm/snippetmodifier.cpp b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.cpp index 127302311f9..127302311f9 100644 --- a/vsm/src/vespa/vsm/vsm/snippetmodifier.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.cpp diff --git a/vsm/src/vespa/vsm/vsm/snippetmodifier.h b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.h index 4718ab8783a..4718ab8783a 100644 --- a/vsm/src/vespa/vsm/vsm/snippetmodifier.h +++ b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.h diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp index 09594832c4a..5507532d4f3 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.cpp @@ -147,18 +147,14 @@ VSMAdapter::configure(const VSMConfigSnapshot & snapshot) LOG(debug, "index highlight spec: '%s'", spec.c_str()); // create dynamic docsum writer - std::unique_ptr<DynamicDocsumWriter> - writer(new DynamicDocsumWriter(resCfg.release(), kwExtractor.release())); + auto writer = std::make_unique<DynamicDocsumWriter>(resCfg.release(), kwExtractor.release()); // configure juniper (used when configuring DynamicDocsumConfig) - std::unique_ptr<juniper::Juniper> juniper; - { - _juniperProps.reset(new JuniperProperties(*juniperrc)); - juniper.reset(new juniper::Juniper(_juniperProps.get(), &_wordFolder)); - } + _juniperProps = std::make_unique<JuniperProperties>(*juniperrc); + auto juniper = std::make_unique<juniper::Juniper>(_juniperProps.get(), &_wordFolder); // create new docsum tools - std::unique_ptr<DocsumTools> docsumTools(new DocsumTools(std::move(writer))); + auto docsumTools = std::make_unique<DocsumTools>(std::move(writer)); docsumTools->setJuniper(std::move(juniper)); // configure dynamic docsum writer diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.h b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h index 6484269353b..6484269353b 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.h +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.h diff --git a/vsm/src/vespa/vsm/vsm/vsm-adapter.hpp b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.hpp index f071dbb2015..f071dbb2015 100644 --- a/vsm/src/vespa/vsm/vsm/vsm-adapter.hpp +++ b/streamingvisitors/src/vespa/vsm/vsm/vsm-adapter.hpp diff --git a/tenant-cd-api/CMakeLists.txt b/tenant-cd-api/CMakeLists.txt index d5155ec34d1..741e0759071 100644 --- a/tenant-cd-api/CMakeLists.txt +++ b/tenant-cd-api/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(tenant-cd-api) +install_jar(tenant-cd-api-jar-with-dependencies.jar) diff --git a/tenant-cd-api/src/main/java/org/apiguardian/api/package-info.java b/tenant-cd-api/src/main/java/org/apiguardian/api/package-info.java new file mode 100644 index 00000000000..85bca21b2ff --- /dev/null +++ b/tenant-cd-api/src/main/java/org/apiguardian/api/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 1, micro = 2)) +package org.apiguardian.api; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/support/package-info.java b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/support/package-info.java new file mode 100644 index 00000000000..087a1f556fc --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/jupiter/api/extension/support/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 5, minor = 8, micro = 1)) +package org.junit.jupiter.api.extension.support; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/platform/commons/annotation/package-info.java b/tenant-cd-api/src/main/java/org/junit/platform/commons/annotation/package-info.java new file mode 100644 index 00000000000..1bdfab887c1 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/platform/commons/annotation/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 8, micro = 1)) +package org.junit.platform.commons.annotation; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/platform/commons/function/package-info.java b/tenant-cd-api/src/main/java/org/junit/platform/commons/function/package-info.java new file mode 100644 index 00000000000..8c577c10b26 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/platform/commons/function/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 8, micro = 1)) +package org.junit.platform.commons.function; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/platform/commons/logging/package-info.java b/tenant-cd-api/src/main/java/org/junit/platform/commons/logging/package-info.java new file mode 100644 index 00000000000..7c8e37cdc72 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/platform/commons/logging/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 8, micro = 1)) +package org.junit.platform.commons.logging; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/platform/commons/package-info.java b/tenant-cd-api/src/main/java/org/junit/platform/commons/package-info.java new file mode 100644 index 00000000000..2b56bf87c9b --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/platform/commons/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 8, micro = 1)) +package org.junit.platform.commons; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/platform/commons/support/package-info.java b/tenant-cd-api/src/main/java/org/junit/platform/commons/support/package-info.java new file mode 100644 index 00000000000..328c88cc1a9 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/platform/commons/support/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 8, micro = 1)) +package org.junit.platform.commons.support; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/junit/platform/commons/util/package-info.java b/tenant-cd-api/src/main/java/org/junit/platform/commons/util/package-info.java new file mode 100644 index 00000000000..811114f4976 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/junit/platform/commons/util/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 8, micro = 1)) +package org.junit.platform.commons.util; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tenant-cd-api/src/main/java/org/opentest4j/package-info.java b/tenant-cd-api/src/main/java/org/opentest4j/package-info.java new file mode 100644 index 00000000000..b7fa6cbf964 --- /dev/null +++ b/tenant-cd-api/src/main/java/org/opentest4j/package-info.java @@ -0,0 +1,9 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/** + * @author jonmv + */ +@ExportPackage(version = @Version(major = 1, minor = 2, micro = 0)) +package org.opentest4j; + +import com.yahoo.osgi.annotation.ExportPackage; +import com.yahoo.osgi.annotation.Version; diff --git a/tsan-suppressions.txt b/tsan-suppressions.txt index f637e1b4ead..00cd7fe9a3a 100644 --- a/tsan-suppressions.txt +++ b/tsan-suppressions.txt @@ -40,3 +40,13 @@ race:llvm::sys::SmartMutex<true>::lock race:re2::Prog::GetDFA race:re2::DFA::DFA + +# Non-TSAN instrumented OpenSSL may trigger warnings when many threads are attempting +# to create TLS connections at once in a newly started process. This appears to be related +# to implicit init-once semantics of internal OpenSSL datastructures, though the TSAN report +# stack frames are truncated and don't show the full picture. +# When a TSAN-instrumented OpenSSL build is used, the error does not manifest itself. +# This implies that it's likely caused by a lack of instrumentation of atomic operations +# (acquire-load in double-checked locking etc). +race:ASN1_STRING_cmp +race:ASN1_STRING_set diff --git a/vbench/CMakeLists.txt b/vbench/CMakeLists.txt index 04a326cfb04..3fb5df8cd20 100644 --- a/vbench/CMakeLists.txt +++ b/vbench/CMakeLists.txt @@ -3,7 +3,6 @@ vespa_define_module( DEPENDS fastos vespalib - staging_vespalib LIBS src/vbench diff --git a/vdslib/CMakeLists.txt b/vdslib/CMakeLists.txt index 4f0e97ae07d..0f8144b99e9 100644 --- a/vdslib/CMakeLists.txt +++ b/vdslib/CMakeLists.txt @@ -4,7 +4,6 @@ vespa_define_module( fastos vespalog vespalib - staging_vespalib config_cloudconfig configdefinitions document diff --git a/vdslib/src/vespa/vdslib/distribution/distribution.h b/vdslib/src/vespa/vdslib/distribution/distribution.h index c09d8400b3a..7b4f0b742e2 100644 --- a/vdslib/src/vespa/vdslib/distribution/distribution.h +++ b/vdslib/src/vespa/vdslib/distribution/distribution.h @@ -8,8 +8,8 @@ #pragma once +#include "group.h" #include <vespa/document/bucket/bucketid.h> -#include <vespa/vdslib/distribution/group.h> #include <vespa/vdslib/state/nodetype.h> #include <vespa/vespalib/util/exception.h> diff --git a/vespa-athenz/CMakeLists.txt b/vespa-athenz/CMakeLists.txt index 5c5fa370796..f5d3dbd3f04 100644 --- a/vespa-athenz/CMakeLists.txt +++ b/vespa-athenz/CMakeLists.txt @@ -1,3 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespa-athenz) +install_jar(vespa-athenz-jar-with-dependencies.jar) install_config_definitions() diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java index 8ffb9331ddb..a4045016b78 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/DefaultZmsClient.java @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.athenz.client.zms; +import com.yahoo.athenz.auth.util.Crypto; +import com.yahoo.security.KeyUtils; import com.yahoo.vespa.athenz.api.AthenzAssertion; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzGroup; @@ -23,6 +25,7 @@ import com.yahoo.vespa.athenz.client.zms.bindings.ResponseListEntity; import com.yahoo.vespa.athenz.client.zms.bindings.RoleEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ServiceEntity; import com.yahoo.vespa.athenz.client.zms.bindings.ServiceListResponseEntity; +import com.yahoo.vespa.athenz.client.zms.bindings.ServicePublicKeyEntity; import com.yahoo.vespa.athenz.client.zms.bindings.StatisticsEntity; import com.yahoo.vespa.athenz.client.zms.bindings.TenancyRequestEntity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; @@ -35,6 +38,7 @@ import org.apache.http.message.BasicHeader; import javax.net.ssl.SSLContext; import java.net.URI; +import java.security.PublicKey; import java.time.Instant; import java.util.Collections; import java.util.HashMap; @@ -356,6 +360,18 @@ public class DefaultZmsClient extends ClientBase implements ZmsClient { } @Override + public void updateServicePublicKey(AthenzService athenzService, String publicKeyId, PublicKey publicKey) { + URI uri = zmsUrl.resolve(String.format("domain/%s/service/%s/publickey/%s", + athenzService.getDomainName(), athenzService.getName(), publicKeyId)); + + ServicePublicKeyEntity entity = new ServicePublicKeyEntity(publicKeyId, Crypto.ybase64EncodeString(KeyUtils.toPem(publicKey))); + HttpUriRequest request = RequestBuilder.put(uri) + .setEntity(toJsonStringEntity(entity)) + .build(); + execute(request, response -> readEntity(response, Void.class)); + } + + @Override public void deleteService(AthenzService athenzService) { URI uri = zmsUrl.resolve(String.format("domain/%s/service/%s", athenzService.getDomainName(), athenzService.getName())); execute(RequestBuilder.delete(uri).build(), response -> readEntity(response, Void.class)); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java index 80a0ddff204..e15af58cb76 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/ZmsClient.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.athenz.api.OAuthCredentials; import java.io.Closeable; +import java.security.PublicKey; import java.time.Instant; import java.util.List; import java.util.Map; @@ -70,6 +71,8 @@ public interface ZmsClient extends Closeable { void createOrUpdateService(AthenzService athenzService); + void updateServicePublicKey(AthenzService athenzService, String publicKeyId, PublicKey publicKey); + void deleteService(AthenzService athenzService); void createRole(AthenzRole role, Map<String, Object> properties); diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServicePublicKeyEntity.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServicePublicKeyEntity.java new file mode 100644 index 00000000000..4767b584661 --- /dev/null +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zms/bindings/ServicePublicKeyEntity.java @@ -0,0 +1,32 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.athenz.client.zms.bindings; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonGetter; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author freva + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public class ServicePublicKeyEntity { + public final String id; + public final String key; + + @JsonCreator + public ServicePublicKeyEntity(@JsonProperty("id") String id, @JsonProperty("key") String key) { + this.id = id; + this.key = key; + } + + @JsonGetter("id") + public String name() { + return id; + } + + @JsonGetter("key") + public String key() { + return key; + } +} diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java index 0c73891bdae..13a61d65d78 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/client/zts/DefaultZtsClient.java @@ -38,7 +38,6 @@ import java.security.KeyPair; import java.security.cert.X509Certificate; import java.time.Duration; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -230,7 +229,7 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { return URI.create(ztsUrl.toString() + '/'); } public static class Builder { - private URI ztsUrl; + private final URI ztsUrl; private ErrorHandler errorHandler = ErrorHandler.empty(); private HostnameVerifier hostnameVerifier = null; private Supplier<SSLContext> sslContextSupplier = null; @@ -260,9 +259,8 @@ public class DefaultZtsClient extends ClientBase implements ZtsClient { } public DefaultZtsClient build() { - if (Objects.isNull(sslContextSupplier)) { - throw new IllegalArgumentException("No ssl context or identity provider available to set up zts client"); - } + if (sslContextSupplier == null) + throw new IllegalArgumentException("No SSL context or identity provider available to set up ZTS client"); return new DefaultZtsClient(ztsUrl, sslContextSupplier, hostnameVerifier, errorHandler); } } diff --git a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzIdentityVerifier.java b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzIdentityVerifier.java index e440d79a159..bc50bcb2bb6 100644 --- a/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzIdentityVerifier.java +++ b/vespa-athenz/src/main/java/com/yahoo/vespa/athenz/tls/AthenzIdentityVerifier.java @@ -24,7 +24,7 @@ public class AthenzIdentityVerifier implements HostnameVerifier { private final Set<AthenzIdentity> allowedIdentities; public AthenzIdentityVerifier(Set<AthenzIdentity> allowedIdentities) { - this.allowedIdentities = allowedIdentities; + this.allowedIdentities = Set.copyOf(allowedIdentities); } @Override diff --git a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java index 0b0e7c5b647..1fcb639589f 100644 --- a/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java +++ b/vespa-documentgen-plugin/src/main/java/com/yahoo/vespa/DocumentGenMojo.java @@ -18,10 +18,10 @@ import com.yahoo.documentmodel.NewDocumentReferenceDataType; import com.yahoo.documentmodel.NewDocumentType; import com.yahoo.documentmodel.OwnedStructDataType; import com.yahoo.documentmodel.VespaDocumentType; -import com.yahoo.searchdefinition.ApplicationBuilder; -import com.yahoo.searchdefinition.Schema; -import com.yahoo.searchdefinition.document.FieldSet; -import com.yahoo.searchdefinition.parser.ParseException; +import com.yahoo.schema.ApplicationBuilder; +import com.yahoo.schema.Schema; +import com.yahoo.schema.document.FieldSet; +import com.yahoo.schema.parser.ParseException; import org.apache.maven.plugin.AbstractMojo; import org.apache.maven.plugins.annotations.LifecyclePhase; import org.apache.maven.plugins.annotations.Mojo; diff --git a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java index 3879067be2e..c619a1ff56b 100644 --- a/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java +++ b/vespa-documentgen-plugin/src/test/java/com/yahoo/vespa/DocumentGenTest.java @@ -4,7 +4,7 @@ package com.yahoo.vespa; import com.yahoo.document.DataType; import com.yahoo.document.StructDataType; import com.yahoo.document.WeightedSetDataType; -import com.yahoo.searchdefinition.Schema; +import com.yahoo.schema.Schema; import org.junit.Test; import java.io.File; diff --git a/vespa-feed-client-cli/CMakeLists.txt b/vespa-feed-client-cli/CMakeLists.txt index 7b0160a62a5..80bfc2234fd 100644 --- a/vespa-feed-client-cli/CMakeLists.txt +++ b/vespa-feed-client-cli/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespa-feed-client-cli) +install_jar(vespa-feed-client-cli-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-feed-client.sh vespa-feed-client bin) install(FILES src/main/resources/logging.properties DESTINATION conf/vespa-feed-client) diff --git a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java index 42abe9e6131..6900c7dc82f 100644 --- a/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java +++ b/vespa-hadoop/src/main/java/com/yahoo/vespa/hadoop/mapreduce/LegacyVespaRecordWriter.java @@ -38,11 +38,10 @@ import java.util.logging.Logger; * @author lesters * @deprecated Replaced by {@link VespaRecordWriter} */ -@SuppressWarnings({"rawtypes", "deprecation"}) @Deprecated -public class LegacyVespaRecordWriter extends RecordWriter { +public class LegacyVespaRecordWriter extends RecordWriter<Object, Object> { - private final static Logger log = Logger.getLogger(LegacyVespaRecordWriter.class.getCanonicalName()); + private final Logger log = Logger.getLogger(getClass().getCanonicalName()); private boolean initialized = false; private com.yahoo.vespa.http.client.FeedClient feedClient; @@ -203,7 +202,7 @@ public class LegacyVespaRecordWriter extends RecordWriter { } - static class ResultCallback implements com.yahoo.vespa.http.client.FeedClient.ResultCallback { + class ResultCallback implements com.yahoo.vespa.http.client.FeedClient.ResultCallback { final VespaCounters counters; public ResultCallback(VespaCounters counters) { diff --git a/vespa-http-client/CMakeLists.txt b/vespa-http-client/CMakeLists.txt index 40d7070eca3..3fb22316a28 100644 --- a/vespa-http-client/CMakeLists.txt +++ b/vespa-http-client/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespa-http-client) +install_jar(vespa-http-client-jar-with-dependencies.jar) diff --git a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java index 0b2c9857ca7..6eb209463fc 100644 --- a/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java +++ b/vespa-maven-plugin/src/main/java/ai/vespa/hosted/plugin/GenerateTestDescriptorMojo.java @@ -14,6 +14,8 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.stream.Stream; +import static java.nio.charset.StandardCharsets.UTF_8; + /** * Generates a test descriptor file based on content of the compiled test classes * @@ -54,7 +56,7 @@ public class GenerateTestDescriptorMojo extends AbstractMojo { try { Path descriptorFile = testClassesDirectory().resolve(TestDescriptor.DEFAULT_FILENAME); Files.createDirectories(descriptorFile.getParent()); - Files.write(descriptorFile, descriptor.toJson().getBytes()); + Files.write(descriptorFile, descriptor.toJson().getBytes(UTF_8)); } catch (IOException e) { throw new MojoExecutionException("Failed to write test descriptor file: " + e.getMessage(), e); } diff --git a/vespa-osgi-testrunner/CMakeLists.txt b/vespa-osgi-testrunner/CMakeLists.txt index e803531f003..294d89871b3 100644 --- a/vespa-osgi-testrunner/CMakeLists.txt +++ b/vespa-osgi-testrunner/CMakeLists.txt @@ -1,3 +1,3 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespa-osgi-testrunner) +install_jar(vespa-osgi-testrunner-jar-with-dependencies.jar) install_config_definitions() diff --git a/vespa-osgi-testrunner/pom.xml b/vespa-osgi-testrunner/pom.xml index de1b5b4024b..4fa651a6930 100644 --- a/vespa-osgi-testrunner/pom.xml +++ b/vespa-osgi-testrunner/pom.xml @@ -21,11 +21,16 @@ <version>${project.version}</version> <scope>provided</scope> </dependency> - + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>hosted-api</artifactId> + <version>${project.version}</version> + <scope>compile</scope> + </dependency> <dependency> <groupId>org.junit.jupiter</groupId> <artifactId>junit-jupiter-engine</artifactId> - <version>5.8.1</version> + <version>5.8.2</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> @@ -36,7 +41,7 @@ <dependency> <groupId>org.junit.platform</groupId> <artifactId>junit-platform-launcher</artifactId> - <version>1.8.1</version> + <version>1.8.2</version> <exclusions> <exclusion> <groupId>org.junit.jupiter</groupId> @@ -44,41 +49,24 @@ </exclusion> </exclusions> </dependency> - - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>tenant-cd-api</artifactId> - <version>${project.version}</version> - <scope>provided</scope> - </dependency> - <dependency> - <groupId>com.yahoo.vespa</groupId> - <artifactId>hosted-api</artifactId> - <version>${project.version}</version> - <scope>compile</scope> - </dependency> <dependency> <groupId>org.fusesource.jansi</groupId> <artifactId>jansi</artifactId> <version>1.18</version> <scope>compile</scope> </dependency> + <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>config-provisioning</artifactId> + <artifactId>tenant-cd-api</artifactId> <version>${project.version}</version> <scope>provided</scope> </dependency> <dependency> <groupId>com.yahoo.vespa</groupId> - <artifactId>testutil</artifactId> + <artifactId>config-provisioning</artifactId> <version>${project.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.mockito</groupId> - <artifactId>mockito-core</artifactId> - <scope>test</scope> + <scope>provided</scope> </dependency> </dependencies> @@ -97,6 +85,13 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <excludedGroups>integration</excludedGroups> + </configuration> + </plugin> </plugins> </build> diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java index 6e3393b2761..15aeef18013 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/AggregateTestRunner.java @@ -4,9 +4,7 @@ package com.yahoo.vespa.testrunner; import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.LogRecord; /** @@ -15,7 +13,8 @@ import java.util.logging.LogRecord; public class AggregateTestRunner implements TestRunner { private final List<TestRunner> wrapped; - private final AtomicInteger current = new AtomicInteger(-1); + private int current = -1; + private final Object monitor = new Object(); private AggregateTestRunner(List<TestRunner> testRunners) { this.wrapped = testRunners; @@ -28,65 +27,77 @@ public class AggregateTestRunner implements TestRunner { @Override public Collection<LogRecord> getLog(long after) { ArrayList<LogRecord> records = new ArrayList<>(); - for (int i = 0; i <= current.get() && i < wrapped.size(); i++) - records.addAll(wrapped.get(i).getLog(after)); - + synchronized (monitor) { + for (int i = 0; i <= current && i < wrapped.size(); i++) + records.addAll(wrapped.get(i).getLog(after)); + } return records; } @Override public Status getStatus() { - if (current.get() == -1) - return Status.NOT_STARTED; - - Status status = Status.NO_TESTS; - for (int i = 0; i <= current.get(); i++) { - if (i == wrapped.size()) - return status; - - Status next = wrapped.get(i).getStatus(); - status = status.ordinal() < next.ordinal() ? status : next; + synchronized (monitor) { + if (current == -1) + return Status.NOT_STARTED; + + Status status = Status.NO_TESTS; + for (int i = 0; i <= current; i++) { + if (i == wrapped.size()) + return status; + + Status next = wrapped.get(i).getStatus(); + status = status.ordinal() < next.ordinal() ? status : next; + } + return Status.RUNNING; } - return Status.RUNNING; } @Override public CompletableFuture<?> test(Suite suite, byte[] config) { - if (0 <= current.get() && current.get() < wrapped.size()) - throw new IllegalStateException("Tests already running, should not attempt to start now"); - - current.set(-1); - CompletableFuture<?> aggregate = new CompletableFuture<>(); - CompletableFuture<?> vessel = CompletableFuture.completedFuture(null); - runNext(suite, config, vessel, aggregate); - return aggregate; + synchronized (monitor) { + if (0 <= current && current < wrapped.size()) + throw new IllegalStateException("Tests already running, should not attempt to start now"); + + current = -1; + CompletableFuture<?> aggregate = new CompletableFuture<>(); + CompletableFuture<?> vessel = CompletableFuture.completedFuture(null); + runNext(suite, config, vessel, aggregate); + return aggregate; + } } private void runNext(Suite suite, byte[] config, CompletableFuture<?> vessel, CompletableFuture<?> aggregate) { vessel.whenComplete((__, ___) -> { - int next = current.incrementAndGet(); - if (next == wrapped.size()) - aggregate.complete(null); - else - runNext(suite, config, wrapped.get(next).test(suite, config), aggregate); + synchronized (monitor) { + if (++current < wrapped.size()) + runNext(suite, config, wrapped.get(current).test(suite, config), aggregate); + else + aggregate.complete(null); + } }); } @Override public TestReport getReport() { - return wrapped.stream().map(TestRunner::getReport).filter(Objects::nonNull) - .reduce(AggregateTestRunner::merge).orElse(null); + TestReport report = null; + synchronized (monitor) { + for (int i = 0; i < current && i < wrapped.size(); i++) + report = merge(report, wrapped.get(i).getReport()); + } + return report; } static TestReport merge(TestReport first, TestReport second) { - return TestReport.builder() - .withAbortedCount(first.abortedCount + second.abortedCount) - .withFailedCount(first.failedCount + second.failedCount) - .withIgnoredCount(first.ignoredCount + second.ignoredCount) - .withSuccessCount(first.successCount + second.successCount) - .withFailures(merged(first.failures, second.failures)) - .withLogs(merged(first.logLines, second.logLines)) - .build(); + return first == null ? second + : second == null ? first + : TestReport.builder() + .withAbortedCount(first.abortedCount + second.abortedCount) + .withFailedCount(first.failedCount + second.failedCount) + .withIgnoredCount(first.ignoredCount + second.ignoredCount) + .withSuccessCount(first.successCount + second.successCount) + .withFailures(merged(first.failures, second.failures)) + .withLogs(merged(first.logLines, second.logLines)) + .build(); } static <T> List<T> merged(List<T> first, List<T> second) { diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java index 90b3f972a3a..c01c9b571e0 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/JunitRunner.java @@ -4,29 +4,21 @@ package com.yahoo.vespa.testrunner; import ai.vespa.cloud.Environment; import ai.vespa.cloud.SystemInfo; import ai.vespa.cloud.Zone; -import ai.vespa.hosted.api.TestDescriptor; import ai.vespa.hosted.cd.InconclusiveTestException; import ai.vespa.hosted.cd.internal.TestRuntimeProvider; -import com.yahoo.component.annotation.Inject; import com.yahoo.component.AbstractComponent; -import com.yahoo.io.IOUtils; +import com.yahoo.component.annotation.Inject; import com.yahoo.jdisc.application.OsgiFramework; import com.yahoo.vespa.defaults.Defaults; import org.junit.jupiter.engine.JupiterTestEngine; -import org.junit.platform.engine.UniqueId; import org.junit.platform.engine.discovery.DiscoverySelectors; -import org.junit.platform.launcher.Launcher; import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestExecutionListener; import org.junit.platform.launcher.core.LauncherConfig; import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; import org.junit.platform.launcher.core.LauncherFactory; import org.junit.platform.launcher.listeners.SummaryGeneratingListener; -import org.osgi.framework.Bundle; -import org.osgi.framework.BundleContext; -import java.io.IOException; -import java.net.URL; -import java.nio.charset.Charset; import java.util.Collection; import java.util.List; import java.util.Optional; @@ -34,11 +26,13 @@ import java.util.SortedMap; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentSkipListMap; import java.util.concurrent.ExecutionException; +import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.LogRecord; import java.util.logging.Logger; -import java.util.stream.Collectors; -import java.util.stream.Stream; + +import static java.util.stream.Collectors.toList; /** * @author mortent @@ -48,8 +42,9 @@ public class JunitRunner extends AbstractComponent implements TestRunner { private static final Logger logger = Logger.getLogger(JunitRunner.class.getName()); private final SortedMap<Long, LogRecord> logRecords = new ConcurrentSkipListMap<>(); - private final BundleContext bundleContext; private final TestRuntimeProvider testRuntimeProvider; + private final Function<Suite, List<Class<?>>> classLoader; + private final BiConsumer<LauncherDiscoveryRequest, TestExecutionListener[]> testExecutor; private volatile CompletableFuture<TestReport> execution; @Inject @@ -57,45 +52,21 @@ public class JunitRunner extends AbstractComponent implements TestRunner { JunitTestRunnerConfig config, TestRuntimeProvider testRuntimeProvider, SystemInfo systemInfo) { - this.testRuntimeProvider = testRuntimeProvider; - this.bundleContext = getUnrestrictedBundleContext(osgiFramework); - uglyHackSetCredentialsRootSystemProperty(config, systemInfo.zone()); - } + this(testRuntimeProvider, + new TestBundleLoader(osgiFramework)::loadTestClasses, + (discoveryRequest, listeners) -> LauncherFactory.create(LauncherConfig.builder() + .addTestEngines(new JupiterTestEngine()) + .build()).execute(discoveryRequest, listeners)); - // Hack to retrieve bundle context that allows access to other bundles - private static BundleContext getUnrestrictedBundleContext(OsgiFramework framework) { - try { - BundleContext restrictedBundleContext = framework.bundleContext(); - var field = restrictedBundleContext.getClass().getDeclaredField("wrapped"); - field.setAccessible(true); - return (BundleContext) field.get(restrictedBundleContext); - } catch (ReflectiveOperationException e) { - throw new RuntimeException(e); - } - } - - // TODO(bjorncs|tokle) Propagate credentials root without system property. Ideally move knowledge about path to test-runtime implementations - private static void uglyHackSetCredentialsRootSystemProperty(JunitTestRunnerConfig config, Zone zone) { - Optional<String> credentialsRoot; - if (config.useAthenzCredentials()) { - credentialsRoot = Optional.of(Defaults.getDefaults().underVespaHome("var/vespa/sia")); - } else if (zone.environment() != Environment.prod){ - // Only set credentials in non-prod zones where not available - credentialsRoot = Optional.of(config.artifactsPath().toString()); - } else { - credentialsRoot = Optional.empty(); - } - credentialsRoot.ifPresent(root -> System.setProperty("vespa.test.credentials.root", root)); + uglyHackSetCredentialsRootSystemProperty(config, systemInfo.zone()); } - private static TestDescriptor.TestCategory toCategory(TestRunner.Suite testProfile) { - switch(testProfile) { - case SYSTEM_TEST: return TestDescriptor.TestCategory.systemtest; - case STAGING_SETUP_TEST: return TestDescriptor.TestCategory.stagingsetuptest; - case STAGING_TEST: return TestDescriptor.TestCategory.stagingtest; - case PRODUCTION_TEST: return TestDescriptor.TestCategory.productiontest; - default: throw new RuntimeException("Unknown test profile: " + testProfile.name()); - } + JunitRunner(TestRuntimeProvider testRuntimeProvider, + Function<Suite, List<Class<?>>> classLoader, + BiConsumer<LauncherDiscoveryRequest, TestExecutionListener[]> testExecutor) { + this.classLoader = classLoader; + this.testExecutor = testExecutor; + this.testRuntimeProvider = testRuntimeProvider; } @Override @@ -105,19 +76,8 @@ public class JunitRunner extends AbstractComponent implements TestRunner { } try { logRecords.clear(); - Optional<Bundle> testBundle = findTestBundle(); - if (testBundle.isEmpty()) { - execution = CompletableFuture.completedFuture(null); - return execution; - } - testRuntimeProvider.initialize(testConfig); - Optional<TestDescriptor> testDescriptor = loadTestDescriptor(testBundle.get()); - if (testDescriptor.isEmpty()) { - throw new RuntimeException("Could not find test descriptor"); - } - execution = CompletableFuture.supplyAsync(() -> launchJunit(loadClasses(testBundle.get(), testDescriptor.get(), toCategory(suite)), - suite == Suite.PRODUCTION_TEST)); + execution = CompletableFuture.supplyAsync(() -> launchJunit(suite)); } catch (Exception e) { execution = CompletableFuture.completedFuture(createReportWithFailedInitialization(e)); } @@ -136,89 +96,45 @@ public class JunitRunner extends AbstractComponent implements TestRunner { .build(); } - private Optional<Bundle> findTestBundle() { - return Stream.of(bundleContext.getBundles()) - .filter(this::isTestBundle) - .findAny(); - } - private boolean isTestBundle(Bundle bundle) { - var testBundleHeader = bundle.getHeaders().get("X-JDisc-Test-Bundle-Version"); - return testBundleHeader != null && ! testBundleHeader.isBlank(); - } - - private Optional<TestDescriptor> loadTestDescriptor(Bundle bundle) { - URL resource = bundle.getEntry(TestDescriptor.DEFAULT_FILENAME); - TestDescriptor testDescriptor; - try { - var jsonDescriptor = IOUtils.readAll(resource.openStream(), Charset.defaultCharset()).trim(); - testDescriptor = TestDescriptor.fromJsonString(jsonDescriptor); - logger.info("Test classes in bundle: " + testDescriptor); - return Optional.of(testDescriptor); - } catch (IOException e) { - return Optional.empty(); - } - } - - private List<Class<?>> loadClasses(Bundle bundle, TestDescriptor testDescriptor, TestDescriptor.TestCategory testCategory) { - List<Class<?>> testClasses = testDescriptor.getConfiguredTests(testCategory).stream() - .map(className -> loadClass(bundle, className)) - .collect(Collectors.toList()); - - StringBuffer buffer = new StringBuffer(); - testClasses.forEach(cl -> buffer.append("\t").append(cl.toString()).append(" / ").append(cl.getClassLoader().toString()).append("\n")); - logger.info("Loaded testClasses: \n" + buffer); - return testClasses; - } + private TestReport launchJunit(Suite suite) { + List<Class<?>> testClasses = classLoader.apply(suite); + if (testClasses == null) + return null; - private Class<?> loadClass(Bundle bundle, String className) { - try { - return bundle.loadClass(className); - } catch (ClassNotFoundException e) { - throw new RuntimeException("Could not find class: " + className + " in bundle " + bundle.getSymbolicName(), e); - } - } - - private TestReport launchJunit(List<Class<?>> testClasses, boolean isProductionTest) { + VespaJunitLogListener logListener = new VespaJunitLogListener(record -> logRecords.put(record.getSequenceNumber(), record)); + SummaryGeneratingListener summaryListener = new SummaryGeneratingListener(); LauncherDiscoveryRequest discoveryRequest = LauncherDiscoveryRequestBuilder.request() - .selectors(testClasses.stream().map(DiscoverySelectors::selectClass).collect(Collectors.toList())) - .build(); - - var launcherConfig = LauncherConfig.builder() - .addTestEngines(new JupiterTestEngine()) - - .build(); - Launcher launcher = LauncherFactory.create(launcherConfig); + .selectors(testClasses.stream() + .map(DiscoverySelectors::selectClass) + .collect(toList())) + .build(); - // Create log listener: - var logListener = new VespaJunitLogListener(record -> logRecords.put(record.getSequenceNumber(), record)); - // Create a summary listener: - var summaryListener = new SummaryGeneratingListener(); - launcher.registerTestExecutionListeners(logListener, summaryListener); + testExecutor.accept(discoveryRequest, new TestExecutionListener[] { logListener, summaryListener }); - // Execute request - launcher.execute(discoveryRequest); var report = summaryListener.getSummary(); var failures = report.getFailures().stream() - .map(failure -> { - TestReport.trimStackTraces(failure.getException(), JunitRunner.class.getName()); - return new TestReport.Failure(VespaJunitLogListener.toString(failure.getTestIdentifier().getUniqueIdObject()), - failure.getException()); - }) - .collect(Collectors.toList()); - long inconclusive = isProductionTest ? failures.stream() - .filter(failure -> failure.exception() instanceof InconclusiveTestException) - .count() - : 0; + .map(failure -> { + TestReport.trimStackTraces(failure.getException(), JunitRunner.class.getName()); + return new TestReport.Failure(VespaJunitLogListener.toString(failure.getTestIdentifier().getUniqueIdObject()), + failure.getException()); + }) + .collect(toList()); + + // TODO: move to aggregator. + long inconclusive = suite == Suite.PRODUCTION_TEST ? failures.stream() + .filter(failure -> failure.exception() instanceof InconclusiveTestException) + .count() + : 0; return TestReport.builder() - .withSuccessCount(report.getTestsSucceededCount()) - .withAbortedCount(report.getTestsAbortedCount()) - .withIgnoredCount(report.getTestsSkippedCount()) - .withFailedCount(report.getTestsFailedCount() - inconclusive) - .withInconclusiveCount(inconclusive) - .withFailures(failures) - .withLogs(logRecords.values()) - .build(); + .withSuccessCount(report.getTestsSucceededCount()) + .withAbortedCount(report.getTestsAbortedCount()) + .withIgnoredCount(report.getTestsSkippedCount()) + .withFailedCount(report.getTestsFailedCount() - inconclusive) + .withInconclusiveCount(inconclusive) + .withFailures(failures) + .withLogs(logRecords.values()) + .build(); } @Override @@ -254,4 +170,18 @@ public class JunitRunner extends AbstractComponent implements TestRunner { } } + // TODO(bjorncs|tokle) Propagate credentials root without system property. Ideally move knowledge about path to test-runtime implementations + private static void uglyHackSetCredentialsRootSystemProperty(JunitTestRunnerConfig config, Zone zone) { + Optional<String> credentialsRoot; + if (config.useAthenzCredentials()) { + credentialsRoot = Optional.of(Defaults.getDefaults().underVespaHome("var/vespa/sia")); + } else if (zone.environment() != Environment.prod){ + // Only set credentials in non-prod zones where not available + credentialsRoot = Optional.of(config.artifactsPath().toString()); + } else { + credentialsRoot = Optional.empty(); + } + credentialsRoot.ifPresent(root -> System.setProperty("vespa.test.credentials.root", root)); + } + } diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestBundleLoader.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestBundleLoader.java new file mode 100644 index 00000000000..3c7c83e3eda --- /dev/null +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestBundleLoader.java @@ -0,0 +1,111 @@ +package com.yahoo.vespa.testrunner; + +import ai.vespa.hosted.api.TestDescriptor; +import com.yahoo.io.IOUtils; +import com.yahoo.jdisc.application.OsgiFramework; +import com.yahoo.vespa.testrunner.TestRunner.Suite; +import org.osgi.framework.Bundle; +import org.osgi.framework.BundleContext; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.Optional; +import java.util.logging.Logger; +import java.util.stream.Stream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.toList; + +/** + * @author mortent + */ +class TestBundleLoader { + + private static final Logger logger = Logger.getLogger(TestBundleLoader.class.getName()); + + private final BundleContext bundleContext; + + public TestBundleLoader(OsgiFramework osgi) { + this.bundleContext = getUnrestrictedBundleContext(osgi); + } + + List<Class<?>> loadTestClasses(Suite suite) { + Optional<Bundle> testBundle = findTestBundle(); + if (testBundle.isEmpty()) + return null; + + Optional<TestDescriptor> testDescriptor = loadTestDescriptor(testBundle.get()); + if (testDescriptor.isEmpty()) + throw new RuntimeException("Could not find test descriptor"); + + return loadClasses(testBundle.get(), testDescriptor.get(), toCategory(suite)); + } + + private Optional<Bundle> findTestBundle() { + return Stream.of(bundleContext.getBundles()).filter(this::isTestBundle).findAny(); + } + + private boolean isTestBundle(Bundle bundle) { + String testBundleHeader = bundle.getHeaders().get("X-JDisc-Test-Bundle-Version"); + return testBundleHeader != null && ! testBundleHeader.isBlank(); + } + + private static Optional<TestDescriptor> loadTestDescriptor(Bundle bundle) { + URL resource = bundle.getEntry(TestDescriptor.DEFAULT_FILENAME); + TestDescriptor testDescriptor; + try { + var jsonDescriptor = IOUtils.readAll(resource.openStream(), UTF_8).trim(); + testDescriptor = TestDescriptor.fromJsonString(jsonDescriptor); + logger.info("Test classes in bundle: " + testDescriptor); + return Optional.of(testDescriptor); + } + catch (IOException e) { + return Optional.empty(); + } + } + + private List<Class<?>> loadClasses(Bundle bundle, TestDescriptor testDescriptor, TestDescriptor.TestCategory testCategory) { + List<Class<?>> testClasses = testDescriptor.getConfiguredTests(testCategory).stream() + .map(className -> loadClass(bundle, className)) + .collect(toList()); + + StringBuffer buffer = new StringBuffer(); + testClasses.forEach(cl -> buffer.append("\t").append(cl.toString()).append(" / ").append(cl.getClassLoader().toString()).append("\n")); + logger.info("Loaded testClasses: \n" + buffer); + return testClasses; + } + + private Class<?> loadClass(Bundle bundle, String className) { + try { + return bundle.loadClass(className); + } + catch (ClassNotFoundException e) { + throw new RuntimeException("Could not find class: " + className + " in bundle " + bundle.getSymbolicName(), e); + } + } + + private static TestDescriptor.TestCategory toCategory(TestRunner.Suite testProfile) { + switch(testProfile) { + case SYSTEM_TEST: return TestDescriptor.TestCategory.systemtest; + case STAGING_SETUP_TEST: return TestDescriptor.TestCategory.stagingsetuptest; + case STAGING_TEST: return TestDescriptor.TestCategory.stagingtest; + case PRODUCTION_TEST: return TestDescriptor.TestCategory.productiontest; + default: throw new RuntimeException("Unknown test profile: " + testProfile.name()); + } + } + + // Hack to retrieve bundle context that allows access to other bundles + private static BundleContext getUnrestrictedBundleContext(OsgiFramework framework) { + try { + BundleContext restrictedBundleContext = framework.bundleContext(); + var field = restrictedBundleContext.getClass().getDeclaredField("wrapped"); + field.setAccessible(true); + return (BundleContext) field.get(restrictedBundleContext); + } + catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + +} diff --git a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java index bcb865c55ff..747005f467d 100644 --- a/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java +++ b/vespa-osgi-testrunner/src/main/java/com/yahoo/vespa/testrunner/TestReport.java @@ -40,7 +40,7 @@ public class TestReport { } public TestRunner.Status status() { - return failedCount > 0 ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : successCount > 0 ? SUCCESS : NO_TESTS; + return (failures.size() > 0 || failedCount > 0) ? FAILURE : inconclusiveCount > 0 ? INCONCLUSIVE : successCount > 0 ? SUCCESS : NO_TESTS; } public static Builder builder(){ diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java new file mode 100644 index 00000000000..4c4e5bf3e36 --- /dev/null +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/test/samples/FailingInstantiationTest.java @@ -0,0 +1,14 @@ +package com.yahoo.vespa.test.samples; + +import ai.vespa.hosted.cd.SystemTest; +import org.junit.jupiter.api.Test; + +@SystemTest +public class FailingInstantiationTest { + + final int i = Integer.parseInt(""); + + @Test + void test() { } + +} diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java index 14e139b3fbf..8bd72d35737 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/AggregateTestRunnerTest.java @@ -136,8 +136,10 @@ class AggregateTestRunnerTest { .build(); first.report = report; assertSame(report, runner.getReport()); - second.report = report; + assertSame(report, runner.getReport()); + + second.future.complete(null); TestReport merged = runner.getReport(); assertEquals(List.of(record1, record1), merged.logLines); assertEquals(List.of(failure, failure), merged.failures); @@ -145,6 +147,7 @@ class AggregateTestRunnerTest { assertEquals(8, merged.ignoredCount); assertEquals(4, merged.failedCount); assertEquals(2, merged.abortedCount); + } @Test @@ -171,9 +174,9 @@ class AggregateTestRunnerTest { catch (Exception e) { TestReport.trimStackTraces(e, "org.junit.platform.launcher.core.SessionPerRequestLauncher"); assertEquals("java.lang.RuntimeException: java.lang.RuntimeException: inner\n" + - "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:168)\n" + + "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:171)\n" + "Caused by: java.lang.RuntimeException: inner\n" + - "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:165)\n", + "\tat com.yahoo.vespa.testrunner.AggregateTestRunnerTest.testStackTrimming(AggregateTestRunnerTest.java:168)\n", ExceptionUtils.getStackTraceAsString(e)); } } diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java new file mode 100644 index 00000000000..64a4c3dbc80 --- /dev/null +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/JunitRunnerTest.java @@ -0,0 +1,103 @@ +package com.yahoo.vespa.testrunner; + +import com.yahoo.vespa.test.samples.FailingInstantiationTest; +import com.yahoo.vespa.testrunner.TestRunner.Status; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.engine.JupiterTestEngine; +import org.junit.platform.engine.EngineExecutionListener; +import org.junit.platform.engine.TestDescriptor; +import org.junit.platform.engine.TestEngine; +import org.junit.platform.engine.TestExecutionResult; +import org.junit.platform.engine.reporting.ReportEntry; +import org.junit.platform.launcher.LauncherDiscoveryRequest; +import org.junit.platform.launcher.TestExecutionListener; +import org.junit.platform.launcher.TestIdentifier; +import org.junit.platform.launcher.TestPlan; +import org.junit.platform.launcher.core.EngineDiscoveryOrchestrator; +import org.junit.platform.launcher.core.EngineExecutionOrchestrator; +import org.junit.platform.launcher.core.LauncherDiscoveryResult; + +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicReference; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.platform.launcher.core.EngineDiscoveryOrchestrator.Phase.EXECUTION; + +/** + * @author jonmv + */ +class JunitRunnerTest { + + @Test + void test() throws ExecutionException, InterruptedException { + AtomicReference<byte[]> testRuntime = new AtomicReference<>(); + JunitRunner runner = new JunitRunner(testRuntime::set, + __ -> List.of(FailingInstantiationTest.class), + this::execute); + + runner.test(null, null).get(); + assertEquals(Status.FAILURE, runner.getStatus()); + assertEquals(0, runner.getReport().successCount); + assertEquals("java.lang.NumberFormatException: For input string: \"\"", + runner.getReport().failures.get(0).exception().toString()); + } + + + // For some inane reason, the JUnit test framework makes it impossible to simply launch a new instance of itself + // from inside a unit test (run by itself) in the standard way, so all this kludge is necessary to work around that. + void execute(LauncherDiscoveryRequest discoveryRequest, TestExecutionListener... listeners) { + TestEngine testEngine = new JupiterTestEngine(); + LauncherDiscoveryResult discoveryResult = new EngineDiscoveryOrchestrator(Set.of(testEngine), Set.of()).discover(discoveryRequest, EXECUTION); + TestDescriptor engineTestDescriptor = discoveryResult.getEngineTestDescriptor(testEngine); + TestPlan plan = TestPlan.from(List.of(engineTestDescriptor), discoveryRequest.getConfigurationParameters()); + for (TestExecutionListener listener : listeners) listener.testPlanExecutionStarted(plan); + new EngineExecutionOrchestrator().execute(discoveryResult, new ExecutionListenerAdapter(plan, listeners)); + for (TestExecutionListener listener : listeners) listener.testPlanExecutionFinished(plan); + } + + static class ExecutionListenerAdapter implements EngineExecutionListener { + + private final TestPlan plan; + private final List<TestExecutionListener> listeners; + + public ExecutionListenerAdapter(TestPlan plan, TestExecutionListener... listeners) { + this.plan = plan; + this.listeners = List.of(listeners); + } + + private TestIdentifier getTestIdentifier(TestDescriptor testDescriptor) { + return plan.getTestIdentifier(testDescriptor.getUniqueId().toString()); + } + + @Override public void dynamicTestRegistered(TestDescriptor testDescriptor) { + TestIdentifier id = TestIdentifier.from(testDescriptor); + plan.addInternal(id); + for (TestExecutionListener listener : listeners) + listener.dynamicTestRegistered(id); + } + + @Override public void executionSkipped(TestDescriptor testDescriptor, String reason) { + for (TestExecutionListener listener : listeners) + listener.executionSkipped(getTestIdentifier(testDescriptor), reason); + } + + @Override public void executionStarted(TestDescriptor testDescriptor) { + for (TestExecutionListener listener : listeners) + listener.executionStarted(getTestIdentifier(testDescriptor)); + } + + @Override public void executionFinished(TestDescriptor testDescriptor, TestExecutionResult testExecutionResult) { + for (TestExecutionListener listener : listeners) + listener.executionFinished(getTestIdentifier(testDescriptor), testExecutionResult); + } + + @Override public void reportingEntryPublished(TestDescriptor testDescriptor, ReportEntry entry) { + for (TestExecutionListener listener : listeners) + listener.reportingEntryPublished(getTestIdentifier(testDescriptor), entry); + } + + } + +} diff --git a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java index 49dd2b20797..49fadebe58b 100644 --- a/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java +++ b/vespa-osgi-testrunner/src/test/java/com/yahoo/vespa/testrunner/TestRunnerHandlerTest.java @@ -5,7 +5,8 @@ import com.yahoo.component.ComponentId; import com.yahoo.component.provider.ComponentRegistry; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; -import com.yahoo.test.json.JsonTestHelper; +import com.yahoo.vespa.testrunner.TestRunner.Status; +import com.yahoo.vespa.testrunner.TestRunner.Suite; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -21,11 +22,10 @@ import java.util.logging.LogRecord; import java.util.stream.Collectors; import static com.yahoo.jdisc.http.HttpRequest.Method.GET; +import static com.yahoo.slime.SlimeUtils.jsonToSlimeOrThrow; +import static com.yahoo.slime.SlimeUtils.toJsonBytes; import static java.nio.charset.StandardCharsets.UTF_8; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.anyLong; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; /** * @author mortent @@ -51,16 +51,18 @@ class TestRunnerHandlerTest { .withFailures(List.of(new TestReport.Failure("Foo.bar()", exception))) .withLogs(logRecords).build(); - aggregateRunner = AggregateTestRunner.of(List.of(new MockJunitRunner(TestRunner.Status.SUCCESS, testReport))); + aggregateRunner = AggregateTestRunner.of(List.of(new MockRunner(TestRunner.Status.SUCCESS, testReport))); testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), aggregateRunner); } @Test public void createsCorrectTestReport() throws IOException { + aggregateRunner.test(Suite.SYSTEM_TEST, new byte[0]); HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/report", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); - JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"summary\":{\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"inconclusive\":5,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"00:00:12.000 Tests started\"]}"); + assertEquals(new String(toJsonBytes(jsonToSlimeOrThrow("{\"summary\":{\"success\":1,\"failed\":2,\"ignored\":3,\"aborted\":4,\"inconclusive\":5,\"failures\":[{\"testName\":\"Foo.bar()\",\"testError\":\"org.junit.ComparisonFailure: expected:<foo> but was:<bar>\",\"exception\":\"java.lang.RuntimeException: org.junit.ComparisonFailure: expected:<foo> but was:<bar>\\n\\tat Foo.bar(Foo.java:1123)\\n\"}]},\"output\":[\"00:00:12.000 Tests started\"]}").get(), false), UTF_8), + new String(toJsonBytes(jsonToSlimeOrThrow(out.toByteArray()).get(), false), UTF_8)); } @Test @@ -71,7 +73,8 @@ class TestRunnerHandlerTest { HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); ByteArrayOutputStream out = new ByteArrayOutputStream(); response.render(out); - JsonTestHelper.assertJsonEquals(out.toString(UTF_8), "{\"logRecords\":[{\"id\":0,\"at\":12000,\"type\":\"info\",\"message\":\"Tests started\"}]}"); + assertEquals(new String(toJsonBytes(jsonToSlimeOrThrow("{\"logRecords\":[{\"id\":0,\"at\":12000,\"type\":\"info\",\"message\":\"Tests started\"}]}").get(), false), UTF_8), + new String(toJsonBytes(jsonToSlimeOrThrow(out.toByteArray()).get(), false), UTF_8)); // Should not get old log response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log?after=0", GET)); @@ -82,11 +85,9 @@ class TestRunnerHandlerTest { @Test public void returnsEmptyResponsesWhenReportNotReady() throws IOException { - TestRunner testRunner = mock(TestRunner.class); - when(testRunner.getReport()).thenReturn(null); - testRunnerHandler = new TestRunnerHandler( - Executors.newSingleThreadExecutor(), - ComponentRegistry.singleton(new ComponentId("runner"), testRunner)); + testRunnerHandler = new TestRunnerHandler(Executors.newSingleThreadExecutor(), + ComponentRegistry.singleton(new ComponentId("runner"), + new MockRunner(Status.NOT_STARTED, null))); { HttpResponse response = testRunnerHandler.handle(HttpRequest.createTestRequest("http://localhost:1234/tester/v1/log", GET)); @@ -111,12 +112,12 @@ class TestRunnerHandlerTest { return logRecord; } - private static class MockJunitRunner implements TestRunner { + private static class MockRunner implements TestRunner { private final TestRunner.Status status; private final TestReport testReport; - public MockJunitRunner(TestRunner.Status status, TestReport testReport) { + public MockRunner(TestRunner.Status status, TestReport testReport) { this.status = status; this.testReport = testReport; @@ -129,9 +130,10 @@ class TestRunnerHandlerTest { @Override public Collection<LogRecord> getLog(long after) { - return getReport().logLines().stream() - .filter(entry -> entry.getSequenceNumber() > after) - .collect(Collectors.toList()); + return getReport() == null ? List.of() + : getReport().logLines().stream() + .filter(entry -> entry.getSequenceNumber() > after) + .collect(Collectors.toList()); } @Override diff --git a/vespa-testrunner-components/CMakeLists.txt b/vespa-testrunner-components/CMakeLists.txt index 7548a05b5bf..4378f8d971f 100644 --- a/vespa-testrunner-components/CMakeLists.txt +++ b/vespa-testrunner-components/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact(vespa-testrunner-components) -install_fat_java_artifact(vespa-testrunner-components) +install_jar(vespa-testrunner-components.jar) +install_jar(vespa-testrunner-components-jar-with-dependencies.jar) install_config_definitions() diff --git a/vespa_feed_perf/CMakeLists.txt b/vespa_feed_perf/CMakeLists.txt index 094ad04cbea..e7339795fcd 100644 --- a/vespa_feed_perf/CMakeLists.txt +++ b/vespa_feed_perf/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespa_feed_perf) +install_jar(vespa_feed_perf-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-feed-perf vespa-feed-perf bin) diff --git a/vespa_jersey2/CMakeLists.txt b/vespa_jersey2/CMakeLists.txt index f75d54991fc..ced1db89e30 100644 --- a/vespa_jersey2/CMakeLists.txt +++ b/vespa_jersey2/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact_dependencies(vespa_jersey2) +install_jar_dependencies(vespa_jersey2) diff --git a/vespabase/src/start-cbinaries.sh b/vespabase/src/start-cbinaries.sh index 2f0a923b074..6a6720e7d9c 100755 --- a/vespabase/src/start-cbinaries.sh +++ b/vespabase/src/start-cbinaries.sh @@ -152,7 +152,9 @@ configure_vespa_malloc () { tryfile="${VESPA_HOME}/${pre}/${suf}" if [ -f "$tryfile" ]; then LD_PRELOAD="$tryfile" - log_debug_message "Using LD_PRELOAD='$tryfile'" + if [ "$VESPA_LOAD_CODE_AS_HUGEPAGES" ]; then + LD_PRELOAD="$LD_PRELOAD:${VESPA_HOME}/${pre}/vespa/malloc/libvespa_load_as_huge.so" + fi if [ "$VESPA_USE_HUGEPAGES" ]; then export VESPA_MALLOC_HUGEPAGES="$VESPA_USE_HUGEPAGES" log_debug_message "enabling hugepages for '$0-bin'." @@ -168,6 +170,8 @@ configure_huge_pages configure_use_madvise configure_vespa_malloc +log_debug_message "Using LD_PRELOAD='$LD_PRELOAD'" + if $no_valgrind ; then numactl=$(get_numa_ctl_cmd) ulimit -c unlimited diff --git a/vespaclient-container-plugin/CMakeLists.txt b/vespaclient-container-plugin/CMakeLists.txt index 57f0847d64a..e1b21cc39f9 100644 --- a/vespaclient-container-plugin/CMakeLists.txt +++ b/vespaclient-container-plugin/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespaclient-container-plugin) +install_jar(vespaclient-container-plugin-jar-with-dependencies.jar) diff --git a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java index eaed95b97e1..9f3c536a59a 100644 --- a/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java +++ b/vespaclient-container-plugin/src/test/java/com/yahoo/document/restapi/resource/DocumentV1ApiTest.java @@ -47,7 +47,7 @@ import com.yahoo.messagebus.StaticThrottlePolicy; import com.yahoo.messagebus.Trace; import com.yahoo.messagebus.TraceNode; import com.yahoo.metrics.simple.MetricReceiver; -import com.yahoo.searchdefinition.derived.Deriver; +import com.yahoo.schema.derived.Deriver; import com.yahoo.slime.Inspector; import com.yahoo.slime.JsonFormat; import com.yahoo.slime.SlimeUtils; diff --git a/vespaclient-java/CMakeLists.txt b/vespaclient-java/CMakeLists.txt index c8f48a8b332..2fb6d0154bb 100644 --- a/vespaclient-java/CMakeLists.txt +++ b/vespaclient-java/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(vespaclient-java) +install_jar(vespaclient-java-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-document-statistics.sh vespa-document-statistics bin) vespa_install_script(src/main/sh/vespa-stat.sh vespa-stat bin) diff --git a/vespaclient/CMakeLists.txt b/vespaclient/CMakeLists.txt index 2a43ea804a3..a427198a22c 100644 --- a/vespaclient/CMakeLists.txt +++ b/vespaclient/CMakeLists.txt @@ -9,7 +9,6 @@ vespa_define_module( document documentapi vespalib - staging_vespalib LIBS src/vespa/vespaclient/clusterlist diff --git a/vespajlib/CMakeLists.txt b/vespajlib/CMakeLists.txt index a22e8c636e1..6d137d0c010 100644 --- a/vespajlib/CMakeLists.txt +++ b/vespajlib/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_java_artifact(vespajlib) +install_jar(vespajlib.jar) diff --git a/vespajlib/abi-spec.json b/vespajlib/abi-spec.json index f7be61946ba..654042372cf 100644 --- a/vespajlib/abi-spec.json +++ b/vespajlib/abi-spec.json @@ -1273,6 +1273,18 @@ ], "fields": [] }, + "com.yahoo.tensor.TensorAddress$PartialBuilder": { + "superClass": "com.yahoo.tensor.TensorAddress$Builder", + "interfaces": [], + "attributes": [ + "public" + ], + "methods": [ + "public void <init>(com.yahoo.tensor.TensorType)", + "public com.yahoo.tensor.TensorAddress$Builder copy()" + ], + "fields": [] + }, "com.yahoo.tensor.TensorAddress": { "superClass": "java.lang.Object", "interfaces": [ diff --git a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java index d9ab67d6c5f..92bdfb2b3a4 100644 --- a/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java +++ b/vespajlib/src/main/java/com/yahoo/tensor/TensorAddress.java @@ -61,8 +61,10 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { @Override public int hashCode() { int result = 1; - for (int i = 0; i < size(); i++) - result = 31 * result + label(i).hashCode(); + for (int i = 0; i < size(); i++) { + if (label(i) != null) + result = 31 * result + label(i).hashCode(); + } return result; } @@ -73,7 +75,7 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { TensorAddress other = (TensorAddress)o; if (other.size() != this.size()) return false; for (int i = 0; i < this.size(); i++) - if ( ! this.label(i).equals(other.label(i))) + if ( ! Objects.equals(this.label(i), other.label(i))) return false; return true; } @@ -171,8 +173,8 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { /** Builder of a tensor address */ public static class Builder { - private final TensorType type; - private final String[] labels; + final TensorType type; + final String[] labels; public Builder(TensorType type) { this(type, new String[type.dimensions().size()]); @@ -184,14 +186,16 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { } /** - * Adds the label to the only dimension of this. + * Adds the label to the only mapped dimension of this. * * @throws IllegalArgumentException if this does not have exactly one dimension */ public Builder add(String label) { - if (type.rank() != 1) - throw new IllegalArgumentException("Cannot add a label without explicit dimension to a tensor of type " + type); - add(type.dimensions().get(0).name(), label); + var mappedSubtype = type.mappedSubtype(); + if (mappedSubtype.rank() != 1) + throw new IllegalArgumentException("Cannot add a label without explicit dimension to a tensor of type " + + type + ": Must have exactly one sparse dimension"); + add(mappedSubtype.dimensions().get(0).name(), label); return this; } @@ -218,14 +222,39 @@ public abstract class TensorAddress implements Comparable<TensorAddress> { /** Returns the type of the tensor this address is being built for. */ public TensorType type() { return type; } - public TensorAddress build() { + void validate() { for (int i = 0; i < labels.length; i++) if (labels[i] == null) throw new IllegalArgumentException("Missing a label for dimension " + type.dimensions().get(i).name() + " for " + type); + } + + public TensorAddress build() { + validate(); return TensorAddress.of(labels); } } + /** Builder of an address to a subset of the dimensions of a tensor type */ + public static class PartialBuilder extends Builder { + + public PartialBuilder(TensorType type) { + super(type); + } + + private PartialBuilder(TensorType type, String[] labels) { + super(type, labels); + } + + /** Creates a copy of this which can be modified separately */ + public Builder copy() { + return new PartialBuilder(type, Arrays.copyOf(labels, labels.length)); + } + + @Override + void validate() { } + + } + } diff --git a/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java index 93bfa3f0dde..62ec6efe3c1 100644 --- a/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java +++ b/vespajlib/src/main/java/com/yahoo/vespa/objects/Ids.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.objects; /** * This is a class containing the global ids that are given out. - * Must be in sync with version for c++ in staging_vespalib/src/vespalib/objects/ids.h + * Must be in sync with version for c++ in vespalib/src/vespalib/objects/ids.h * * @author baldersheim */ diff --git a/vespalib/CMakeLists.txt b/vespalib/CMakeLists.txt index 7796ce7df28..69bd709c613 100644 --- a/vespalib/CMakeLists.txt +++ b/vespalib/CMakeLists.txt @@ -1,4 +1,9 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") +set(VESPALIB_DIRECTIO_TESTDIR src/tests/directio) +set(VESPALIB_PROCESS_MEMORY_STATS_TESTDIR src/tests/util/process_memory_stats) +endif() + vespa_define_module( DEPENDS fastos @@ -13,6 +18,7 @@ vespa_define_module( src/apps/make_fixture_macros src/apps/vespa-detect-hostname src/apps/vespa-drop-file-from-cache + src/apps/vespa-resource-limits src/apps/vespa-tsan-digest src/apps/vespa-validate-hostname @@ -24,16 +30,20 @@ vespa_define_module( src/tests/arrayref src/tests/assert src/tests/barrier + src/tests/benchmark src/tests/benchmark_timer + src/tests/bits src/tests/box src/tests/btree src/tests/btree/btree_store src/tests/btree/btree-scan-speed src/tests/btree/btree-stress + src/tests/clock src/tests/component src/tests/compress src/tests/compression src/tests/cpu_usage + src/tests/crc src/tests/crypto src/tests/data/databuffer src/tests/data/input_reader @@ -52,10 +62,12 @@ vespa_define_module( src/tests/datastore/unique_store src/tests/datastore/unique_store_dictionary src/tests/datastore/unique_store_string_allocator + ${VESPALIB_DIRECTIO_TESTDIR} src/tests/detect_type_benchmark src/tests/dotproduct src/tests/drop-file-from-cache src/tests/dual_merge_director + src/tests/encoding/base64 src/tests/eventbarrier src/tests/exception_classes src/tests/executor @@ -63,18 +75,24 @@ vespa_define_module( src/tests/explore_modern_cpp src/tests/false src/tests/fiddle + src/tests/fileheader + src/tests/floatingpointtype src/tests/fuzzy src/tests/gencnt + src/tests/growablebytebuffer src/tests/guard src/tests/host_name src/tests/hwaccelrated src/tests/io/fileutil src/tests/io/mapped_file_input src/tests/issue + src/tests/json src/tests/latch src/tests/left_right_heap src/tests/make_fixture_macros src/tests/memory + src/tests/memorydatastore + src/tests/metrics src/tests/net/async_resolver src/tests/net/crypto_socket src/tests/net/selector @@ -89,9 +107,13 @@ vespa_define_module( src/tests/net/tls/protocol_snooping src/tests/net/tls/transport_options src/tests/nice + src/tests/objects/identifiable src/tests/objects/nbostream + src/tests/objects/objectdump + src/tests/objects/objectselection src/tests/optimized src/tests/overload + src/tests/polymorphicarray src/tests/portal src/tests/portal/handle_manager src/tests/portal/http_request @@ -99,12 +121,18 @@ vespa_define_module( src/tests/printable src/tests/priority_queue src/tests/process + ${VESPALIB_PROCESS_MEMORY_STATS_TESTDIR} + src/tests/programoptions src/tests/random src/tests/referencecounter src/tests/regex src/tests/rendezvous src/tests/require src/tests/runnable_pair + src/tests/rusage + src/tests/sequencedtaskexecutor + src/tests/shutdownguard + src/tests/singleexecutor src/tests/sha1 src/tests/shared_operation_throttler src/tests/shared_string_repo @@ -117,6 +145,7 @@ vespa_define_module( src/tests/small_vector src/tests/spin_lock src/tests/stash + src/tests/state_server src/tests/stllike src/tests/stringfmt src/tests/sync @@ -146,6 +175,7 @@ vespa_define_module( src/tests/typify src/tests/unwind_message src/tests/util/bfloat16 + src/tests/util/cgroup_resource_limits src/tests/util/file_area_freelist src/tests/util/generationhandler src/tests/util/generationhandler_stress @@ -161,7 +191,10 @@ vespa_define_module( src/tests/visit_ranges src/tests/invokeservice src/tests/wakeup + src/tests/xmlserializable src/tests/zcurve + src/tests/fastlib/io + src/tests/fastlib/text LIBS src/vespa/vespalib @@ -171,12 +204,15 @@ vespa_define_module( src/vespa/vespalib/data src/vespa/vespalib/data/slime src/vespa/vespalib/datastore + src/vespa/vespalib/encoding src/vespa/vespalib/fuzzy src/vespa/vespalib/geo src/vespa/vespalib/hwaccelrated src/vespa/vespalib/io src/vespa/vespalib/locale + src/vespa/vespalib/metrics src/vespa/vespalib/net + src/vespa/vespalib/net/http src/vespa/vespalib/net/tls src/vespa/vespalib/net/tls/impl src/vespa/vespalib/objects @@ -190,4 +226,7 @@ vespa_define_module( src/vespa/vespalib/time src/vespa/vespalib/trace src/vespa/vespalib/util + src/vespa/fastlib/io + src/vespa/fastlib/text + src/vespa/fastlib/text/apps ) diff --git a/vespalib/src/apps/vespa-resource-limits/.gitignore b/vespalib/src/apps/vespa-resource-limits/.gitignore new file mode 100644 index 00000000000..58a2938f452 --- /dev/null +++ b/vespalib/src/apps/vespa-resource-limits/.gitignore @@ -0,0 +1 @@ +/vespa-resource-limits diff --git a/vespalib/src/apps/vespa-resource-limits/CMakeLists.txt b/vespalib/src/apps/vespa-resource-limits/CMakeLists.txt new file mode 100644 index 00000000000..566b23058cf --- /dev/null +++ b/vespalib/src/apps/vespa-resource-limits/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_vespa-resource-limits_app + SOURCES + resource_limits.cpp + OUTPUT_NAME vespa-resource-limits + INSTALL bin + DEPENDS + vespalib +) diff --git a/vespalib/src/apps/vespa-resource-limits/resource_limits.cpp b/vespalib/src/apps/vespa-resource-limits/resource_limits.cpp new file mode 100644 index 00000000000..59bea7d0fd8 --- /dev/null +++ b/vespalib/src/apps/vespa-resource-limits/resource_limits.cpp @@ -0,0 +1,13 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/util/resource_limits.h> +#include <iostream> + +int +main(int, char**) +{ + auto limits = vespalib::ResourceLimits::create(); + std::cout << "Memory limit: " << limits.memory() << '\n' << + "Cpu limit: " << limits.cpu() << std::endl; + return 0; +} diff --git a/vespalib/src/tests/array/.gitignore b/vespalib/src/tests/array/.gitignore index 952bc0ecdf2..dd07c7d9777 100644 --- a/vespalib/src/tests/array/.gitignore +++ b/vespalib/src/tests/array/.gitignore @@ -1,3 +1,4 @@ /sort_benchmark /allocinarray_benchmark vespalib_array_test_app +vespalib_sort_benchmark_app diff --git a/vespalib/src/tests/array/CMakeLists.txt b/vespalib/src/tests/array/CMakeLists.txt index 18cb5c2b95e..fae7b32cd7e 100644 --- a/vespalib/src/tests/array/CMakeLists.txt +++ b/vespalib/src/tests/array/CMakeLists.txt @@ -6,3 +6,10 @@ vespa_add_executable(vespalib_array_test_app TEST vespalib ) vespa_add_test(NAME vespalib_array_test_app COMMAND vespalib_array_test_app) +vespa_add_executable(vespalib_sort_benchmark_app + SOURCES + sort_benchmark.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_sort_benchmark_app COMMAND vespalib_sort_benchmark_app BENCHMARK) diff --git a/staging_vespalib/src/tests/array/sort_benchmark.cpp b/vespalib/src/tests/array/sort_benchmark.cpp index 5d8a1efaa7a..5d8a1efaa7a 100644 --- a/staging_vespalib/src/tests/array/sort_benchmark.cpp +++ b/vespalib/src/tests/array/sort_benchmark.cpp diff --git a/vespalib/src/tests/benchmark/.gitignore b/vespalib/src/tests/benchmark/.gitignore new file mode 100644 index 00000000000..3280cb17888 --- /dev/null +++ b/vespalib/src/tests/benchmark/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +benchmark_test +vespalib_benchmark_test_app diff --git a/vespalib/src/tests/benchmark/CMakeLists.txt b/vespalib/src/tests/benchmark/CMakeLists.txt new file mode 100644 index 00000000000..7003a5c4183 --- /dev/null +++ b/vespalib/src/tests/benchmark/CMakeLists.txt @@ -0,0 +1,12 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_benchmark_test_app + SOURCES + benchmark.cpp + testbase.cpp + DEPENDS + vespalib + EXTERNAL_DEPENDS + ${VESPA_GLIBC_RT_LIB} +) +vespa_add_test(NAME vespalib_benchmark_test NO_VALGRIND COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_test.sh BENCHMARK + DEPENDS vespalib_benchmark_test_app) diff --git a/staging_vespalib/src/tests/benchmark/benchmark.cpp b/vespalib/src/tests/benchmark/benchmark.cpp index f1e69758c8c..f1e69758c8c 100644 --- a/staging_vespalib/src/tests/benchmark/benchmark.cpp +++ b/vespalib/src/tests/benchmark/benchmark.cpp diff --git a/vespalib/src/tests/benchmark/benchmark_test.sh b/vespalib/src/tests/benchmark/benchmark_test.sh new file mode 100755 index 00000000000..28dc6b518be --- /dev/null +++ b/vespalib/src/tests/benchmark/benchmark_test.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +set -e +TIME=time + +$TIME ./vespalib_benchmark_test_app vespalib::ParamByReferenceVectorInt 200000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ParamByValueVectorInt 4000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ParamByReferenceVectorString 30000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ParamByValueVectorString 40 1 +$TIME ./vespalib_benchmark_test_app vespalib::ReturnByReferenceVectorString 10 1 +$TIME ./vespalib_benchmark_test_app vespalib::ReturnByValueVectorString 10 1 +$TIME ./vespalib_benchmark_test_app vespalib::ReturnByValueMultiVectorString 10 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockSystem 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockGToD 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockGToD 20000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockREALTIME 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockMONOTONIC 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockMONOTONIC_RAW 1000 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockPROCESS_CPUTIME_ID 2500 1 +$TIME ./vespalib_benchmark_test_app vespalib::ClockTHREAD_CPUTIME_ID 2500 1 +$TIME ./vespalib_benchmark_test_app vespalib::CreateVespalibString 20000 1 diff --git a/staging_vespalib/src/tests/benchmark/testbase.cpp b/vespalib/src/tests/benchmark/testbase.cpp index 6b5f8d7d627..6b5f8d7d627 100644 --- a/staging_vespalib/src/tests/benchmark/testbase.cpp +++ b/vespalib/src/tests/benchmark/testbase.cpp diff --git a/staging_vespalib/src/tests/benchmark/testbase.h b/vespalib/src/tests/benchmark/testbase.h index 95621f52471..95621f52471 100644 --- a/staging_vespalib/src/tests/benchmark/testbase.h +++ b/vespalib/src/tests/benchmark/testbase.h diff --git a/vespalib/src/tests/bits/.gitignore b/vespalib/src/tests/bits/.gitignore new file mode 100644 index 00000000000..b5330fc2580 --- /dev/null +++ b/vespalib/src/tests/bits/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +bits_test +vespalib_bits_test_app diff --git a/vespalib/src/tests/bits/CMakeLists.txt b/vespalib/src/tests/bits/CMakeLists.txt new file mode 100644 index 00000000000..f63196bc489 --- /dev/null +++ b/vespalib/src/tests/bits/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_bits_test_app TEST + SOURCES + bits_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_bits_test_app COMMAND vespalib_bits_test_app) diff --git a/staging_vespalib/src/tests/bits/bits_test.cpp b/vespalib/src/tests/bits/bits_test.cpp index 47d691c739d..47d691c739d 100644 --- a/staging_vespalib/src/tests/bits/bits_test.cpp +++ b/vespalib/src/tests/bits/bits_test.cpp diff --git a/vespalib/src/tests/clock/.gitignore b/vespalib/src/tests/clock/.gitignore new file mode 100644 index 00000000000..96861fcc5d3 --- /dev/null +++ b/vespalib/src/tests/clock/.gitignore @@ -0,0 +1,5 @@ +.depend +Makefile +clock_test +vespalib_clock_test_app +vespalib_clock_benchmark_app diff --git a/vespalib/src/tests/clock/CMakeLists.txt b/vespalib/src/tests/clock/CMakeLists.txt new file mode 100644 index 00000000000..d3ee3178163 --- /dev/null +++ b/vespalib/src/tests/clock/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_clock_benchmark_app TEST + SOURCES + clock_benchmark.cpp + DEPENDS + vespalib +) +vespa_add_executable(vespalib_clock_test_app TEST + SOURCES + clock_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_clock_test_app COMMAND vespalib_clock_test_app) diff --git a/staging_vespalib/src/tests/clock/clock_benchmark.cpp b/vespalib/src/tests/clock/clock_benchmark.cpp index 249add4bc1a..249add4bc1a 100644 --- a/staging_vespalib/src/tests/clock/clock_benchmark.cpp +++ b/vespalib/src/tests/clock/clock_benchmark.cpp diff --git a/staging_vespalib/src/tests/clock/clock_test.cpp b/vespalib/src/tests/clock/clock_test.cpp index f2de085da84..f2de085da84 100644 --- a/staging_vespalib/src/tests/clock/clock_test.cpp +++ b/vespalib/src/tests/clock/clock_test.cpp diff --git a/vespalib/src/tests/crc/.gitignore b/vespalib/src/tests/crc/.gitignore new file mode 100644 index 00000000000..cd64c20e0a4 --- /dev/null +++ b/vespalib/src/tests/crc/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +crc_test +vespalib_crc_test_app diff --git a/vespalib/src/tests/crc/CMakeLists.txt b/vespalib/src/tests/crc/CMakeLists.txt new file mode 100644 index 00000000000..30adfd131f1 --- /dev/null +++ b/vespalib/src/tests/crc/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_crc_test_app TEST + SOURCES + crc_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_crc_test_app COMMAND vespalib_crc_test_app boost) diff --git a/staging_vespalib/src/tests/crc/crc_test.cpp b/vespalib/src/tests/crc/crc_test.cpp index 8afeed487ee..8afeed487ee 100644 --- a/staging_vespalib/src/tests/crc/crc_test.cpp +++ b/vespalib/src/tests/crc/crc_test.cpp diff --git a/vespalib/src/tests/directio/.gitignore b/vespalib/src/tests/directio/.gitignore new file mode 100644 index 00000000000..ad19022dfc3 --- /dev/null +++ b/vespalib/src/tests/directio/.gitignore @@ -0,0 +1 @@ +vespalib_directio_test_app diff --git a/vespalib/src/tests/directio/CMakeLists.txt b/vespalib/src/tests/directio/CMakeLists.txt new file mode 100644 index 00000000000..41a8dca85b9 --- /dev/null +++ b/vespalib/src/tests/directio/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_directio_test_app TEST + SOURCES + directio.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_directio_test_app COMMAND vespalib_directio_test_app) diff --git a/staging_vespalib/src/tests/directio/directio.cpp b/vespalib/src/tests/directio/directio.cpp index 9bc4f244711..77374f6f926 100644 --- a/staging_vespalib/src/tests/directio/directio.cpp +++ b/vespalib/src/tests/directio/directio.cpp @@ -19,7 +19,7 @@ TEST("that DirectIOException propagates the correct information.") { } TEST("that DirectIOException is thrown on unaligned buf.") { - FastOS_File f("staging_vespalib_directio_test_app"); + FastOS_File f("vespalib_directio_test_app"); f.EnableDirectIO(); EXPECT_TRUE(f.OpenReadOnly()); DataBuffer buf(10000, 4_Ki); @@ -37,7 +37,7 @@ TEST("that DirectIOException is thrown on unaligned buf.") { } TEST("that DirectIOException is thrown on unaligned offset.") { - FastOS_File f("staging_vespalib_directio_test_app"); + FastOS_File f("vespalib_directio_test_app"); f.EnableDirectIO(); EXPECT_TRUE(f.OpenReadOnly()); DataBuffer buf(10000, 4_Ki); diff --git a/staging_vespalib/src/tests/encoding/.gitignore b/vespalib/src/tests/encoding/.gitignore index a3e9c375723..a3e9c375723 100644 --- a/staging_vespalib/src/tests/encoding/.gitignore +++ b/vespalib/src/tests/encoding/.gitignore diff --git a/vespalib/src/tests/encoding/base64/.gitignore b/vespalib/src/tests/encoding/base64/.gitignore new file mode 100644 index 00000000000..bd63ed7e5cb --- /dev/null +++ b/vespalib/src/tests/encoding/base64/.gitignore @@ -0,0 +1 @@ +vespalib_base64_test_app diff --git a/vespalib/src/tests/encoding/base64/CMakeLists.txt b/vespalib/src/tests/encoding/base64/CMakeLists.txt new file mode 100644 index 00000000000..e2bb5d83fbe --- /dev/null +++ b/vespalib/src/tests/encoding/base64/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_base64_test_app TEST + SOURCES + base64_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_base64_test_app COMMAND vespalib_base64_test_app) diff --git a/staging_vespalib/src/tests/encoding/base64/base64_test.cpp b/vespalib/src/tests/encoding/base64/base64_test.cpp index 295aad7ffdd..295aad7ffdd 100644 --- a/staging_vespalib/src/tests/encoding/base64/base64_test.cpp +++ b/vespalib/src/tests/encoding/base64/base64_test.cpp diff --git a/fastlib/src/vespa/fastlib/io/tests/.gitignore b/vespalib/src/tests/fastlib/io/.gitignore index 816281ccbfb..816281ccbfb 100644 --- a/fastlib/src/vespa/fastlib/io/tests/.gitignore +++ b/vespalib/src/tests/fastlib/io/.gitignore diff --git a/fastlib/src/vespa/fastlib/io/tests/CMakeLists.txt b/vespalib/src/tests/fastlib/io/CMakeLists.txt index ae1e81a6170..345b3456bbf 100644 --- a/fastlib/src/vespa/fastlib/io/tests/CMakeLists.txt +++ b/vespalib/src/tests/fastlib/io/CMakeLists.txt @@ -3,6 +3,6 @@ vespa_add_executable(fastlib_bufferedfiletest_app TEST SOURCES bufferedfiletest.cpp DEPENDS - fastlib_io + vespalib ) vespa_add_test(NAME fastlib_bufferedfiletest_app COMMAND fastlib_bufferedfiletest_app) diff --git a/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp b/vespalib/src/tests/fastlib/io/bufferedfiletest.cpp index 8aa9b943419..8aa9b943419 100644 --- a/fastlib/src/vespa/fastlib/io/tests/bufferedfiletest.cpp +++ b/vespalib/src/tests/fastlib/io/bufferedfiletest.cpp diff --git a/fastlib/src/vespa/fastlib/text/tests/.gitignore b/vespalib/src/tests/fastlib/text/.gitignore index 8134602778d..8134602778d 100644 --- a/fastlib/src/vespa/fastlib/text/tests/.gitignore +++ b/vespalib/src/tests/fastlib/text/.gitignore diff --git a/fastlib/src/vespa/fastlib/text/tests/CMakeLists.txt b/vespalib/src/tests/fastlib/text/CMakeLists.txt index bcb7be76aac..690da0a3d80 100644 --- a/fastlib/src/vespa/fastlib/text/tests/CMakeLists.txt +++ b/vespalib/src/tests/fastlib/text/CMakeLists.txt @@ -3,23 +3,13 @@ vespa_add_executable(fastlib_unicodeutiltest_app TEST SOURCES unicodeutiltest.cpp DEPENDS - fastlib_text - fastlib_fast_testsuite + vespalib ) vespa_add_test(NAME fastlib_unicodeutiltest_app NO_VALGRIND COMMAND fastlib_unicodeutiltest_app) -vespa_add_executable(fastlib_latintokenizertest_app TEST - SOURCES - latintokenizertest.cpp - DEPENDS - fastlib_text - fastlib_fast_testsuite -) -vespa_add_test(NAME fastlib_latintokenizertest_app NO_VALGRIND COMMAND fastlib_latintokenizertest_app) vespa_add_executable(fastlib_wordfolderstest_app TEST SOURCES wordfolderstest.cpp DEPENDS - fastlib_text - fastlib_fast_testsuite + vespalib ) vespa_add_test(NAME fastlib_wordfolderstest_app NO_VALGRIND COMMAND fastlib_wordfolderstest_app) diff --git a/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp b/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp new file mode 100644 index 00000000000..d734b3b6aab --- /dev/null +++ b/vespalib/src/tests/fastlib/text/unicodeutiltest.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastlib/text/unicodeutil.h> +#include <vespa/vespalib/testkit/test_kit.h> + +TEST("GetUTF8Char_WrongInput") { + const char *testdata = "ab\xF8"; + + ucs4_t the_char = 0; + + const unsigned char *src = reinterpret_cast<const unsigned char *>(testdata); + while (*src != 0) { + the_char = Fast_UnicodeUtil::GetUTF8Char(src); + } + EXPECT_EQUAL(Fast_UnicodeUtil::_BadUTF8Char, the_char); +} + +TEST("IsTerminalPunctuationChar") { + // test a small selection + + EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar('!')); + EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(',')); + EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar('.')); + EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(':')); + EXPECT_TRUE(Fast_UnicodeUtil::IsTerminalPunctuationChar(';')); + EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar(' ')); + EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar('a')); + EXPECT_FALSE(Fast_UnicodeUtil::IsTerminalPunctuationChar('A')); +} + +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/vespalib/src/tests/fastlib/text/wordfolderstest.cpp b/vespalib/src/tests/fastlib/text/wordfolderstest.cpp new file mode 100644 index 00000000000..b2e05250951 --- /dev/null +++ b/vespalib/src/tests/fastlib/text/wordfolderstest.cpp @@ -0,0 +1,46 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/fastlib/text/normwordfolder.h> +#include <vespa/vespalib/testkit/test_kit.h> + +TEST("NormalizeWordFolderConstruction") { + Fast_NormalizeWordFolder::Setup( + Fast_NormalizeWordFolder::DO_ACCENT_REMOVAL + | Fast_NormalizeWordFolder::DO_KATAKANA_TO_HIRAGANA + | Fast_NormalizeWordFolder::DO_SMALL_TO_NORMAL_KANA + | Fast_NormalizeWordFolder::DO_SHARP_S_SUBSTITUTION + | Fast_NormalizeWordFolder::DO_LIGATURE_SUBSTITUTION + | Fast_NormalizeWordFolder::DO_MULTICHAR_EXPANSION); +} + +TEST("TokenizeAnnotatedUCS4Buffer") { + auto nwf = std::make_unique<Fast_NormalizeWordFolder>(); + const char *testinput = "This is a " + "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB" + " superduperextrafeaturecoolandlongplainword fun " + "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" + "world wide web extra long annotation block" "\xEF\xBF\xBB" + " test\nIt is cool.\n"; + const char *correct[] = { + "this", "is", "a", + "\xEF\xBF\xB9" "café" "\xEF\xBF\xBA" "cafe" "\xEF\xBF\xBB", + "superduperextrafeaturecooland", "fun", + "\xEF\xBF\xB9" "www" "\xEF\xBF\xBA" "world wide web extra lon", + "test", "it", "is", "cool" }; + + const char *teststart = testinput; + const char *testend = testinput + strlen(testinput); + ucs4_t destbuf[32]; + ucs4_t *destbufend = destbuf + 32; + + const char *origstart = testinput; + size_t tokenlen = 0; + + int tokencounter = 0; + while ((teststart = nwf->UCS4Tokenize(teststart, testend, destbuf, destbufend, origstart, tokenlen)) < testend) { + EXPECT_EQUAL(0, Fast_UnicodeUtil::utf8cmp(correct[tokencounter++], destbuf)); + } + +} + +TEST_MAIN() { TEST_RUN_ALL(); }
\ No newline at end of file diff --git a/staging_vespalib/src/tests/fileheader/.gitignore b/vespalib/src/tests/fileheader/.gitignore index 909655ba711..f41a6844d34 100644 --- a/staging_vespalib/src/tests/fileheader/.gitignore +++ b/vespalib/src/tests/fileheader/.gitignore @@ -3,4 +3,4 @@ Makefile fileheader.tmp fileheader_test -staging_vespalib_fileheader_test_app +vespalib_fileheader_test_app diff --git a/vespalib/src/tests/fileheader/CMakeLists.txt b/vespalib/src/tests/fileheader/CMakeLists.txt new file mode 100644 index 00000000000..a58507e818e --- /dev/null +++ b/vespalib/src/tests/fileheader/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_fileheader_test_app TEST + SOURCES + fileheader_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_fileheader_test_app COMMAND vespalib_fileheader_test_app) diff --git a/staging_vespalib/src/tests/fileheader/fileheader.dat b/vespalib/src/tests/fileheader/fileheader.dat Binary files differindex 90660f64b98..90660f64b98 100644 --- a/staging_vespalib/src/tests/fileheader/fileheader.dat +++ b/vespalib/src/tests/fileheader/fileheader.dat diff --git a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp b/vespalib/src/tests/fileheader/fileheader_test.cpp index 21e374e4f62..21e374e4f62 100644 --- a/staging_vespalib/src/tests/fileheader/fileheader_test.cpp +++ b/vespalib/src/tests/fileheader/fileheader_test.cpp diff --git a/vespalib/src/tests/floatingpointtype/.gitignore b/vespalib/src/tests/floatingpointtype/.gitignore new file mode 100644 index 00000000000..abe8249f33a --- /dev/null +++ b/vespalib/src/tests/floatingpointtype/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +floatingpointtype_test +vespalib_floatingpointtype_test_app diff --git a/vespalib/src/tests/floatingpointtype/CMakeLists.txt b/vespalib/src/tests/floatingpointtype/CMakeLists.txt new file mode 100644 index 00000000000..3f0ec8eab69 --- /dev/null +++ b/vespalib/src/tests/floatingpointtype/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_floatingpointtype_test_app TEST + SOURCES + floatingpointtypetest.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_floatingpointtype_test_app COMMAND vespalib_floatingpointtype_test_app) diff --git a/staging_vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp b/vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp index d26385f23bf..d26385f23bf 100644 --- a/staging_vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp +++ b/vespalib/src/tests/floatingpointtype/floatingpointtypetest.cpp diff --git a/vespalib/src/tests/growablebytebuffer/.gitignore b/vespalib/src/tests/growablebytebuffer/.gitignore new file mode 100644 index 00000000000..76218df9168 --- /dev/null +++ b/vespalib/src/tests/growablebytebuffer/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +growablebytebuffer_test +vespalib_growablebytebuffer_test_app diff --git a/vespalib/src/tests/growablebytebuffer/CMakeLists.txt b/vespalib/src/tests/growablebytebuffer/CMakeLists.txt new file mode 100644 index 00000000000..b518206ae56 --- /dev/null +++ b/vespalib/src/tests/growablebytebuffer/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_growablebytebuffer_test_app TEST + SOURCES + growablebytebuffer_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_growablebytebuffer_test_app COMMAND vespalib_growablebytebuffer_test_app) diff --git a/staging_vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp b/vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp index 0a616745023..0a616745023 100644 --- a/staging_vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp +++ b/vespalib/src/tests/growablebytebuffer/growablebytebuffer_test.cpp diff --git a/vespalib/src/tests/json/.gitignore b/vespalib/src/tests/json/.gitignore new file mode 100644 index 00000000000..9918fbce6e8 --- /dev/null +++ b/vespalib/src/tests/json/.gitignore @@ -0,0 +1 @@ +vespalib_json_test_app diff --git a/vespalib/src/tests/json/CMakeLists.txt b/vespalib/src/tests/json/CMakeLists.txt new file mode 100644 index 00000000000..0ea216b189b --- /dev/null +++ b/vespalib/src/tests/json/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_json_test_app TEST + SOURCES + json.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_json_test_app COMMAND vespalib_json_test_app boost) diff --git a/staging_vespalib/src/tests/json/json.cpp b/vespalib/src/tests/json/json.cpp index 1a707ae1776..1a707ae1776 100644 --- a/staging_vespalib/src/tests/json/json.cpp +++ b/vespalib/src/tests/json/json.cpp diff --git a/vespalib/src/tests/memorydatastore/.gitignore b/vespalib/src/tests/memorydatastore/.gitignore new file mode 100644 index 00000000000..6809bff5d3d --- /dev/null +++ b/vespalib/src/tests/memorydatastore/.gitignore @@ -0,0 +1 @@ +vespalib_memorydatastore_test_app diff --git a/vespalib/src/tests/memorydatastore/CMakeLists.txt b/vespalib/src/tests/memorydatastore/CMakeLists.txt new file mode 100644 index 00000000000..65d9231455a --- /dev/null +++ b/vespalib/src/tests/memorydatastore/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_memorydatastore_test_app TEST + SOURCES + memorydatastore.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_memorydatastore_test_app COMMAND vespalib_memorydatastore_test_app) diff --git a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp b/vespalib/src/tests/memorydatastore/memorydatastore.cpp index 1d49b0af91b..1d49b0af91b 100644 --- a/staging_vespalib/src/tests/memorydatastore/memorydatastore.cpp +++ b/vespalib/src/tests/memorydatastore/memorydatastore.cpp diff --git a/vespalib/src/tests/metrics/CMakeLists.txt b/vespalib/src/tests/metrics/CMakeLists.txt new file mode 100644 index 00000000000..6019a6c2d4c --- /dev/null +++ b/vespalib/src/tests/metrics/CMakeLists.txt @@ -0,0 +1,17 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_metrics_test_app TEST + SOURCES + simple_metrics_test.cpp + mock_tick.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_metrics_test_app COMMAND vespalib_metrics_test_app) + +vespa_add_executable(vespalib_stablestore_test_app TEST + SOURCES + stable_store_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_stablestore_test_app COMMAND vespalib_stablestore_test_app) diff --git a/staging_vespalib/src/tests/metrics/mock_tick.cpp b/vespalib/src/tests/metrics/mock_tick.cpp index c16ef25cfe6..c16ef25cfe6 100644 --- a/staging_vespalib/src/tests/metrics/mock_tick.cpp +++ b/vespalib/src/tests/metrics/mock_tick.cpp diff --git a/staging_vespalib/src/tests/metrics/mock_tick.h b/vespalib/src/tests/metrics/mock_tick.h index 4d9f6758537..4d9f6758537 100644 --- a/staging_vespalib/src/tests/metrics/mock_tick.h +++ b/vespalib/src/tests/metrics/mock_tick.h diff --git a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp b/vespalib/src/tests/metrics/simple_metrics_test.cpp index 3006022a43d..3006022a43d 100644 --- a/staging_vespalib/src/tests/metrics/simple_metrics_test.cpp +++ b/vespalib/src/tests/metrics/simple_metrics_test.cpp diff --git a/staging_vespalib/src/tests/metrics/stable_store_test.cpp b/vespalib/src/tests/metrics/stable_store_test.cpp index cead112069f..cead112069f 100644 --- a/staging_vespalib/src/tests/metrics/stable_store_test.cpp +++ b/vespalib/src/tests/metrics/stable_store_test.cpp diff --git a/staging_vespalib/src/tests/objects/.gitignore b/vespalib/src/tests/objects/identifiable/.gitignore index 2b10aff029c..a547ace8ee4 100644 --- a/staging_vespalib/src/tests/objects/.gitignore +++ b/vespalib/src/tests/objects/identifiable/.gitignore @@ -2,4 +2,4 @@ Makefile asciistream_test identifiable_test -staging_vespalib_identifiable_test_app +vespalib_identifiable_test_app diff --git a/vespalib/src/tests/objects/identifiable/CMakeLists.txt b/vespalib/src/tests/objects/identifiable/CMakeLists.txt new file mode 100644 index 00000000000..c4aefa44350 --- /dev/null +++ b/vespalib/src/tests/objects/identifiable/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_identifiable_test_app TEST + SOURCES + identifiable_test.cpp + namedobject.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_identifiable_test_app COMMAND vespalib_identifiable_test_app) diff --git a/staging_vespalib/src/tests/objects/identifiable_test.cpp b/vespalib/src/tests/objects/identifiable/identifiable_test.cpp index bc20a525d23..b3adfbfa9e2 100644 --- a/staging_vespalib/src/tests/objects/identifiable_test.cpp +++ b/vespalib/src/tests/objects/identifiable/identifiable_test.cpp @@ -1,10 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include "namedobject.h" #include <vespa/vespalib/objects/identifiable.hpp> -#include <vespa/vespalib/objects/namedobject.h> #include <vespa/vespalib/objects/nbostream.h> #include <vespa/vespalib/util/exceptions.h> +#include <vespa/vespalib/testkit/testapp.h> using namespace vespalib; diff --git a/staging_vespalib/src/vespa/vespalib/objects/namedobject.cpp b/vespalib/src/tests/objects/identifiable/namedobject.cpp index 3e8d3291177..3e8d3291177 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/namedobject.cpp +++ b/vespalib/src/tests/objects/identifiable/namedobject.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/namedobject.h b/vespalib/src/tests/objects/identifiable/namedobject.h index 784715a66f6..784715a66f6 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/namedobject.h +++ b/vespalib/src/tests/objects/identifiable/namedobject.h diff --git a/vespalib/src/tests/objects/objectdump/.gitignore b/vespalib/src/tests/objects/objectdump/.gitignore new file mode 100644 index 00000000000..6ddd515391d --- /dev/null +++ b/vespalib/src/tests/objects/objectdump/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +objectdump_test +vespalib_objectdump_test_app diff --git a/vespalib/src/tests/objects/objectdump/CMakeLists.txt b/vespalib/src/tests/objects/objectdump/CMakeLists.txt new file mode 100644 index 00000000000..67395998b39 --- /dev/null +++ b/vespalib/src/tests/objects/objectdump/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_objectdump_test_app TEST + SOURCES + objectdump.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_objectdump_test_app COMMAND vespalib_objectdump_test_app) diff --git a/staging_vespalib/src/tests/objectdump/objectdump.cpp b/vespalib/src/tests/objects/objectdump/objectdump.cpp index 812b1e79e17..812b1e79e17 100644 --- a/staging_vespalib/src/tests/objectdump/objectdump.cpp +++ b/vespalib/src/tests/objects/objectdump/objectdump.cpp diff --git a/vespalib/src/tests/objects/objectselection/.gitignore b/vespalib/src/tests/objects/objectselection/.gitignore new file mode 100644 index 00000000000..f6aefd07270 --- /dev/null +++ b/vespalib/src/tests/objects/objectselection/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +objectselection_test +vespalib_objectselection_test_app diff --git a/vespalib/src/tests/objects/objectselection/CMakeLists.txt b/vespalib/src/tests/objects/objectselection/CMakeLists.txt new file mode 100644 index 00000000000..94f43078820 --- /dev/null +++ b/vespalib/src/tests/objects/objectselection/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_objectselection_test_app TEST + SOURCES + objectselection.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_objectselection_test_app COMMAND vespalib_objectselection_test_app) diff --git a/staging_vespalib/src/tests/objectselection/objectselection.cpp b/vespalib/src/tests/objects/objectselection/objectselection.cpp index aa9c841f2dc..aa9c841f2dc 100644 --- a/staging_vespalib/src/tests/objectselection/objectselection.cpp +++ b/vespalib/src/tests/objects/objectselection/objectselection.cpp diff --git a/vespalib/src/tests/polymorphicarray/.gitignore b/vespalib/src/tests/polymorphicarray/.gitignore new file mode 100644 index 00000000000..e0decc87f2c --- /dev/null +++ b/vespalib/src/tests/polymorphicarray/.gitignore @@ -0,0 +1 @@ +vespalib_polymorphicarray_test_app diff --git a/vespalib/src/tests/polymorphicarray/CMakeLists.txt b/vespalib/src/tests/polymorphicarray/CMakeLists.txt new file mode 100644 index 00000000000..14edfbec4b4 --- /dev/null +++ b/vespalib/src/tests/polymorphicarray/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_polymorphicarray_test_app TEST + SOURCES + polymorphicarray_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_polymorphicarray_test_app COMMAND vespalib_polymorphicarray_test_app) diff --git a/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp index d4ec8f3ed7c..d4ec8f3ed7c 100644 --- a/staging_vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp +++ b/vespalib/src/tests/polymorphicarray/polymorphicarray_test.cpp diff --git a/vespalib/src/tests/programoptions/.gitignore b/vespalib/src/tests/programoptions/.gitignore new file mode 100644 index 00000000000..f083a1e093d --- /dev/null +++ b/vespalib/src/tests/programoptions/.gitignore @@ -0,0 +1,4 @@ +.depend +Makefile +programoptions_test +vespalib_programoptions_test_app diff --git a/vespalib/src/tests/programoptions/CMakeLists.txt b/vespalib/src/tests/programoptions/CMakeLists.txt new file mode 100644 index 00000000000..fb2fdb48dd7 --- /dev/null +++ b/vespalib/src/tests/programoptions/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_programoptions_test_app TEST + SOURCES + programoptions_test.cpp + programoptions_testutils.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_programoptions_test_app COMMAND vespalib_programoptions_test_app) diff --git a/staging_vespalib/src/tests/programoptions/programoptions_test.cpp b/vespalib/src/tests/programoptions/programoptions_test.cpp index 4a5be1d1397..4b63eae949b 100644 --- a/staging_vespalib/src/tests/programoptions/programoptions_test.cpp +++ b/vespalib/src/tests/programoptions/programoptions_test.cpp @@ -1,8 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> +#include "programoptions_testutils.h" #include <vespa/vespalib/util/programoptions.h> -#include <vespa/vespalib/util/programoptions_testutils.h> +#include <vespa/vespalib/testkit/testapp.h> #include <iostream> namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.cpp b/vespalib/src/tests/programoptions/programoptions_testutils.cpp index 4f77a01aa02..948413c36db 100644 --- a/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.cpp +++ b/vespalib/src/tests/programoptions/programoptions_testutils.cpp @@ -1,6 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/util/programoptions_testutils.h> +#include "programoptions_testutils.h" namespace vespalib { diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.h b/vespalib/src/tests/programoptions/programoptions_testutils.h index a6f103f3e95..a6f103f3e95 100644 --- a/staging_vespalib/src/vespa/vespalib/util/programoptions_testutils.h +++ b/vespalib/src/tests/programoptions/programoptions_testutils.h diff --git a/vespalib/src/tests/rusage/.gitignore b/vespalib/src/tests/rusage/.gitignore new file mode 100644 index 00000000000..c01c01ed328 --- /dev/null +++ b/vespalib/src/tests/rusage/.gitignore @@ -0,0 +1 @@ +vespalib_rusage_test_app diff --git a/vespalib/src/tests/rusage/CMakeLists.txt b/vespalib/src/tests/rusage/CMakeLists.txt new file mode 100644 index 00000000000..1c1ab85facd --- /dev/null +++ b/vespalib/src/tests/rusage/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_rusage_test_app TEST + SOURCES + rusage_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_rusage_test_app COMMAND vespalib_rusage_test_app) diff --git a/staging_vespalib/src/tests/rusage/rusage_test.cpp b/vespalib/src/tests/rusage/rusage_test.cpp index 7e30f3b968b..7e30f3b968b 100644 --- a/staging_vespalib/src/tests/rusage/rusage_test.cpp +++ b/vespalib/src/tests/rusage/rusage_test.cpp diff --git a/vespalib/src/tests/sequencedtaskexecutor/.gitignore b/vespalib/src/tests/sequencedtaskexecutor/.gitignore new file mode 100644 index 00000000000..3b6f7c74a67 --- /dev/null +++ b/vespalib/src/tests/sequencedtaskexecutor/.gitignore @@ -0,0 +1,4 @@ +vespalib_sequencedtaskexecutor_test_app +vespalib_sequencedtaskexecutor_benchmark_app +vespalib_adaptive_sequenced_executor_test_app +vespalib_foregroundtaskexecutor_test_app diff --git a/vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt b/vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt new file mode 100644 index 00000000000..6a488b3c716 --- /dev/null +++ b/vespalib/src/tests/sequencedtaskexecutor/CMakeLists.txt @@ -0,0 +1,31 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_sequencedtaskexecutor_benchmark_app TEST + SOURCES + sequencedtaskexecutor_benchmark.cpp + DEPENDS + vespalib +) + +vespa_add_executable(vespalib_sequencedtaskexecutor_test_app TEST + SOURCES + sequencedtaskexecutor_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_sequencedtaskexecutor_test_app COMMAND vespalib_sequencedtaskexecutor_test_app) + +vespa_add_executable(vespalib_adaptive_sequenced_executor_test_app TEST + SOURCES + adaptive_sequenced_executor_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_adaptive_sequenced_executor_test_app COMMAND vespalib_adaptive_sequenced_executor_test_app) + +vespa_add_executable(vespalib_foregroundtaskexecutor_test_app TEST + SOURCES + foregroundtaskexecutor_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_foregroundtaskexecutor_test_app COMMAND vespalib_foregroundtaskexecutor_test_app) diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp index 1a458f86232..1a458f86232 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/adaptive_sequenced_executor_test.cpp diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp index 56fb570209c..56fb570209c 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/foregroundtaskexecutor_test.cpp diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp index 0f7c82ef988..0f7c82ef988 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_benchmark.cpp diff --git a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp index 705d6346e8c..705d6346e8c 100644 --- a/staging_vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp +++ b/vespalib/src/tests/sequencedtaskexecutor/sequencedtaskexecutor_test.cpp diff --git a/vespalib/src/tests/shutdownguard/.gitignore b/vespalib/src/tests/shutdownguard/.gitignore new file mode 100644 index 00000000000..c167d4784ca --- /dev/null +++ b/vespalib/src/tests/shutdownguard/.gitignore @@ -0,0 +1 @@ +vespalib_shutdownguard_test_app diff --git a/vespalib/src/tests/shutdownguard/CMakeLists.txt b/vespalib/src/tests/shutdownguard/CMakeLists.txt new file mode 100644 index 00000000000..6714842fbbf --- /dev/null +++ b/vespalib/src/tests/shutdownguard/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_shutdownguard_test_app TEST + SOURCES + shutdownguard_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_shutdownguard_test_app NO_VALGRIND COMMAND vespalib_shutdownguard_test_app) diff --git a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp index 348e9bbd503..348e9bbd503 100644 --- a/staging_vespalib/src/tests/shutdownguard/shutdownguard_test.cpp +++ b/vespalib/src/tests/shutdownguard/shutdownguard_test.cpp diff --git a/vespalib/src/tests/singleexecutor/CMakeLists.txt b/vespalib/src/tests/singleexecutor/CMakeLists.txt new file mode 100644 index 00000000000..3580a91d114 --- /dev/null +++ b/vespalib/src/tests/singleexecutor/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_singleexecutor_test_app TEST + SOURCES + singleexecutor_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_singleexecutor_test_app COMMAND vespalib_singleexecutor_test_app) diff --git a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp index 56352ff3c0d..56352ff3c0d 100644 --- a/staging_vespalib/src/tests/singleexecutor/singleexecutor_test.cpp +++ b/vespalib/src/tests/singleexecutor/singleexecutor_test.cpp diff --git a/vespalib/src/tests/state_server/.gitignore b/vespalib/src/tests/state_server/.gitignore new file mode 100644 index 00000000000..2d36d34e2ec --- /dev/null +++ b/vespalib/src/tests/state_server/.gitignore @@ -0,0 +1 @@ +vespalib_state_server_test_app diff --git a/vespalib/src/tests/state_server/CMakeLists.txt b/vespalib/src/tests/state_server/CMakeLists.txt new file mode 100644 index 00000000000..6d3d762a42b --- /dev/null +++ b/vespalib/src/tests/state_server/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_state_server_test_app TEST + SOURCES + state_server_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_state_server_test_app NO_VALGRIND NO_VALGRIND COMMAND vespalib_state_server_test_app ENVIRONMENT "VESPA_HOME=.") diff --git a/staging_vespalib/src/tests/state_server/state_server_test.cpp b/vespalib/src/tests/state_server/state_server_test.cpp index cb80a44a675..f6e614f213a 100644 --- a/staging_vespalib/src/tests/state_server/state_server_test.cpp +++ b/vespalib/src/tests/state_server/state_server_test.cpp @@ -1,21 +1,17 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <fcntl.h> -#include <stdlib.h> -#include <sys/stat.h> -#include <sys/types.h> - #include <vespa/vespalib/testkit/test_kit.h> -#include <vespa/vespalib/util/host_name.h> -#include <vespa/vespalib/net/state_server.h> -#include <vespa/vespalib/net/simple_health_producer.h> -#include <vespa/vespalib/net/simple_metrics_producer.h> -#include <vespa/vespalib/net/simple_component_config_producer.h> +#include <vespa/vespalib/net/http/state_server.h> +#include <vespa/vespalib/net/http/simple_health_producer.h> +#include <vespa/vespalib/net/http/simple_metrics_producer.h> +#include <vespa/vespalib/net/http/simple_component_config_producer.h> +#include <vespa/vespalib/net/http/state_explorer.h> +#include <vespa/vespalib/net/http/slime_explorer.h> +#include <vespa/vespalib/net/http/generic_state_handler.h> #include <vespa/vespalib/util/stringfmt.h> +#include <vespa/vespalib/util/host_name.h> #include <vespa/vespalib/process/process.h> -#include <vespa/vespalib/net/state_explorer.h> -#include <vespa/vespalib/net/slime_explorer.h> -#include <vespa/vespalib/net/generic_state_handler.h> +#include <sys/stat.h> using namespace vespalib; diff --git a/vespalib/src/tests/stllike/CMakeLists.txt b/vespalib/src/tests/stllike/CMakeLists.txt index 80509c565c6..005ef3d1ed0 100644 --- a/vespalib/src/tests/stllike/CMakeLists.txt +++ b/vespalib/src/tests/stllike/CMakeLists.txt @@ -55,3 +55,17 @@ vespa_add_executable(vespalib_replace_variable_test_app TEST GTest::GTest ) vespa_add_test(NAME vespalib_replace_variable_test_app COMMAND vespalib_replace_variable_test_app) +vespa_add_executable(vespalib_lrucache_test_app TEST + SOURCES + lrucache.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_lrucache_test_app COMMAND vespalib_lrucache_test_app) +vespa_add_executable(vespalib_cache_test_app TEST + SOURCES + cache_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_cache_test_app COMMAND vespalib_cache_test_app) diff --git a/staging_vespalib/src/tests/stllike/cache_test.cpp b/vespalib/src/tests/stllike/cache_test.cpp index 35f04d91510..35f04d91510 100644 --- a/staging_vespalib/src/tests/stllike/cache_test.cpp +++ b/vespalib/src/tests/stllike/cache_test.cpp diff --git a/staging_vespalib/src/tests/stllike/lrucache.cpp b/vespalib/src/tests/stllike/lrucache.cpp index 2cc6f2b4ee8..2cc6f2b4ee8 100644 --- a/staging_vespalib/src/tests/stllike/lrucache.cpp +++ b/vespalib/src/tests/stllike/lrucache.cpp diff --git a/vespalib/src/tests/trace/CMakeLists.txt b/vespalib/src/tests/trace/CMakeLists.txt index a632d419b4b..76cb266f230 100644 --- a/vespalib/src/tests/trace/CMakeLists.txt +++ b/vespalib/src/tests/trace/CMakeLists.txt @@ -7,10 +7,10 @@ vespa_add_executable(vespalib_trace_test_app TEST ) vespa_add_test(NAME vespalib_trace_test_app COMMAND vespalib_trace_test_app) -vespa_add_executable(staging_vespalib_trace_serialization_test_app TEST +vespa_add_executable(vespalib_trace_serialization_test_app TEST SOURCES trace_serialization.cpp DEPENDS vespalib ) -vespa_add_test(NAME staging_vespalib_trace_serialization_test_app COMMAND staging_vespalib_trace_serialization_test_app) +vespa_add_test(NAME vespalib_trace_serialization_test_app COMMAND vespalib_trace_serialization_test_app) diff --git a/vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt b/vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt new file mode 100644 index 00000000000..79414f9ef9f --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_cgroup_resource_limits_test_app TEST + SOURCES + cgroup_resource_limits_test.cpp + DEPENDS + vespalib + GTest::GTest +) +vespa_add_test(NAME vespalib_cgroup_resource_limits_test_app COMMAND vespalib_cgroup_resource_limits_test_app) diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp new file mode 100644 index 00000000000..a4e57f69ab8 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_resource_limits_test.cpp @@ -0,0 +1,75 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vespalib/util/cgroup_resource_limits.h> +#include <vespa/vespalib/util/size_literals.h> + +namespace vespalib { + +class CGroupResourceLimitsTest : public ::testing::Test +{ +protected: + CGroupResourceLimitsTest(); + ~CGroupResourceLimitsTest(); + void check_limits(const std::string &name, const std::optional<uint64_t>& memory_limit, const std::optional<uint32_t>& cpu_limit); +}; + +CGroupResourceLimitsTest::CGroupResourceLimitsTest() = default; +CGroupResourceLimitsTest::~CGroupResourceLimitsTest() = default; + +void +CGroupResourceLimitsTest::check_limits(const std::string &base, const std::optional<uint64_t>& memory_limit, const std::optional<uint32_t>& cpu_limit) +{ + CGroupResourceLimits cg_limits(base + "/cgroup", base + "/self"); + EXPECT_EQ(memory_limit, cg_limits.get_memory_limit()); + EXPECT_EQ(cpu_limit, cg_limits.get_cpu_limit()); +} + +TEST_F(CGroupResourceLimitsTest, no_cgroup) +{ + check_limits("no_cgroup", std::nullopt, std::nullopt); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_host) +{ + check_limits("cgroup_v1_host", 4_Mi, 3); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_host_nested) +{ + check_limits("cgroup_v1_host_nested", 5_Mi, 4); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_host_no_limit) +{ + check_limits("cgroup_v1_host_no_limit", std::nullopt, std::nullopt); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v1_container) +{ + check_limits("cgroup_v1_container", 8_Mi, 5); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_host) +{ + check_limits("cgroup_v2_host", 12_Mi, 7); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_host_nested) +{ + check_limits("cgroup_v2_host_nested", 13_Mi, 8); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_host_no_limit) +{ + check_limits("cgroup_v2_host_no_limit", std::nullopt, std::nullopt); +} + +TEST_F(CGroupResourceLimitsTest, cgroup_v2_container) +{ + check_limits("cgroup_v2_container", 16_Mi, 9); +} + +} + +GTEST_MAIN_RUN_ALL_TESTS() diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us new file mode 100644 index 00000000000..354b2529b29 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/cpu/cpu.cfs_quota_us @@ -0,0 +1 @@ +500000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes new file mode 100644 index 00000000000..16edb4cc19a --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/cgroup/memory/memory.limit_in_bytes @@ -0,0 +1 @@ +8388608 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self new file mode 100644 index 00000000000..ee80ea30458 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_container/self @@ -0,0 +1,2 @@ +1:memory:/group1 +1:cpu,cpuacct:/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us new file mode 100644 index 00000000000..67f9d558228 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/cpu/group2/cpu.cfs_quota_us @@ -0,0 +1 @@ +300000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes new file mode 100644 index 00000000000..3f7803daddf --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/cgroup/memory/group1/memory.limit_in_bytes @@ -0,0 +1 @@ +4194304 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self new file mode 100644 index 00000000000..ee80ea30458 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host/self @@ -0,0 +1,2 @@ +1:memory:/group1 +1:cpu,cpuacct:/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us new file mode 100644 index 00000000000..004d15285e7 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/cpu.cfs_quota_us @@ -0,0 +1 @@ +400000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us new file mode 100644 index 00000000000..354b2529b29 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/cpu/group2/group4/cpu.cfs_quota_us @@ -0,0 +1 @@ +500000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes new file mode 100644 index 00000000000..faefc1fbffc --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/group3/memory.limit_in_bytes @@ -0,0 +1 @@ +6291456 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes new file mode 100644 index 00000000000..062933f0941 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/cgroup/memory/group1/memory.limit_in_bytes @@ -0,0 +1 @@ +5242880 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self new file mode 100644 index 00000000000..98a1f55ba89 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_nested/self @@ -0,0 +1,2 @@ +1:memory:/group1/group3 +1:cpu,cpuacct:/group2/group4 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us new file mode 100644 index 00000000000..f7393e847d3 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_period_us @@ -0,0 +1 @@ +100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us new file mode 100644 index 00000000000..3a2e3f4984a --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/cpu/group2/cpu.cfs_quota_us @@ -0,0 +1 @@ +-1 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes new file mode 100644 index 00000000000..564113cfaff --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/cgroup/memory/group1/memory.limit_in_bytes @@ -0,0 +1 @@ +9223372036854771712 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self new file mode 100644 index 00000000000..ee80ea30458 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v1_host_no_limit/self @@ -0,0 +1,2 @@ +1:memory:/group1 +1:cpu,cpuacct:/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max new file mode 100644 index 00000000000..06f6446c456 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/cpu.max @@ -0,0 +1 @@ +900000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max new file mode 100644 index 00000000000..27f897b99ff --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/cgroup/memory.max @@ -0,0 +1 @@ +16777216 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self new file mode 100644 index 00000000000..f85fa774795 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_container/self @@ -0,0 +1 @@ +0::/group1 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max new file mode 100644 index 00000000000..fd6b2b4e326 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/cpu.max @@ -0,0 +1 @@ +700000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max new file mode 100644 index 00000000000..372ed62fb4b --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/cgroup/group1/memory.max @@ -0,0 +1 @@ +12582912 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self new file mode 100644 index 00000000000..f85fa774795 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host/self @@ -0,0 +1 @@ +0::/group1 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max new file mode 100644 index 00000000000..5c5f99b398b --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/cpu.max @@ -0,0 +1 @@ +800000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max new file mode 100644 index 00000000000..06f6446c456 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/cpu.max @@ -0,0 +1 @@ +900000 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max new file mode 100644 index 00000000000..9812e3d8f08 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/group2/memory.max @@ -0,0 +1 @@ +14680064 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max new file mode 100644 index 00000000000..03b7c4afd21 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/cgroup/group1/memory.max @@ -0,0 +1 @@ +13631488 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self new file mode 100644 index 00000000000..b2c3ecd563b --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_nested/self @@ -0,0 +1 @@ +0::/group1/group2 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max new file mode 100644 index 00000000000..1c1d3e7c303 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/cpu.max @@ -0,0 +1 @@ +max 100000 diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max new file mode 100644 index 00000000000..355295a05af --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/cgroup/group1/memory.max @@ -0,0 +1 @@ +max diff --git a/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self new file mode 100644 index 00000000000..f85fa774795 --- /dev/null +++ b/vespalib/src/tests/util/cgroup_resource_limits/cgroup_v2_host_no_limit/self @@ -0,0 +1 @@ +0::/group1 diff --git a/vespalib/src/tests/util/process_memory_stats/.gitignore b/vespalib/src/tests/util/process_memory_stats/.gitignore new file mode 100644 index 00000000000..81af04ee64f --- /dev/null +++ b/vespalib/src/tests/util/process_memory_stats/.gitignore @@ -0,0 +1,2 @@ +mapfile +vespalib_process_memory_stats_test_app diff --git a/vespalib/src/tests/util/process_memory_stats/CMakeLists.txt b/vespalib/src/tests/util/process_memory_stats/CMakeLists.txt new file mode 100644 index 00000000000..30a0f90d952 --- /dev/null +++ b/vespalib/src/tests/util/process_memory_stats/CMakeLists.txt @@ -0,0 +1,9 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_process_memory_stats_test_app TEST + SOURCES + process_memory_stats_test.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_process_memory_stats_test_app COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/process_memory_stats_test.sh + DEPENDS vespalib_process_memory_stats_test_app) diff --git a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp index 6d0917e6d15..6d0917e6d15 100644 --- a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp +++ b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp diff --git a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh index 6441a5e3039..7fe5261ab2d 100755 --- a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh +++ b/vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.sh @@ -2,5 +2,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. set -e rm -f mapfile -$VALGRIND ./staging_vespalib_process_memory_stats_test_app +$VALGRIND ./vespalib_process_memory_stats_test_app rm -f mapfile diff --git a/vespalib/src/tests/xmlserializable/.gitignore b/vespalib/src/tests/xmlserializable/.gitignore new file mode 100644 index 00000000000..8573da1cd93 --- /dev/null +++ b/vespalib/src/tests/xmlserializable/.gitignore @@ -0,0 +1,4 @@ +*_test +.depend +Makefile +vespalib_xmlserializable_test_app diff --git a/vespalib/src/tests/xmlserializable/CMakeLists.txt b/vespalib/src/tests/xmlserializable/CMakeLists.txt new file mode 100644 index 00000000000..119a1294253 --- /dev/null +++ b/vespalib/src/tests/xmlserializable/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(vespalib_xmlserializable_test_app TEST + SOURCES + xmlserializabletest.cpp + DEPENDS + vespalib +) +vespa_add_test(NAME vespalib_xmlserializable_test_app COMMAND vespalib_xmlserializable_test_app) diff --git a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp b/vespalib/src/tests/xmlserializable/xmlserializabletest.cpp index cc8d61cb7c2..cc8d61cb7c2 100644 --- a/staging_vespalib/src/tests/xmlserializable/xmlserializabletest.cpp +++ b/vespalib/src/tests/xmlserializable/xmlserializabletest.cpp diff --git a/fastlib/src/vespa/fastlib/io/.gitignore b/vespalib/src/vespa/fastlib/io/.gitignore index aa9c3f19188..aa9c3f19188 100644 --- a/fastlib/src/vespa/fastlib/io/.gitignore +++ b/vespalib/src/vespa/fastlib/io/.gitignore diff --git a/fastlib/src/vespa/fastlib/io/CMakeLists.txt b/vespalib/src/vespa/fastlib/io/CMakeLists.txt index 58244af5798..f21cf27b21e 100644 --- a/fastlib/src/vespa/fastlib/io/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/io/CMakeLists.txt @@ -1,7 +1,6 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_io +vespa_add_library(fastlib_io OBJECT SOURCES bufferedfile.cpp - INSTALL lib64 DEPENDS ) diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp index 56ffbf40eca..fa55d0be812 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.cpp +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.cpp @@ -375,7 +375,7 @@ size_t computeBufLen(size_t buflen) Fast_BufferedFile::Fast_BufferedFile(FastOS_FileInterface *file, size_t bufferSize) : FastOS_FileInterface(), _fileleft(static_cast<uint64_t>(-1)), - _buf(vespalib::alloc::Alloc::allocMMap(computeBufLen(bufferSize))), + _buf(vespalib::alloc::Alloc::alloc(computeBufLen(bufferSize))), _bufi(nullptr), _bufe(nullptr), _filepos(0), diff --git a/fastlib/src/vespa/fastlib/io/bufferedfile.h b/vespalib/src/vespa/fastlib/io/bufferedfile.h index 48f90262ad9..48f90262ad9 100644 --- a/fastlib/src/vespa/fastlib/io/bufferedfile.h +++ b/vespalib/src/vespa/fastlib/io/bufferedfile.h diff --git a/fastlib/src/vespa/fastlib/text/.gitignore b/vespalib/src/vespa/fastlib/text/.gitignore index c452e2714be..c452e2714be 100644 --- a/fastlib/src/vespa/fastlib/text/.gitignore +++ b/vespalib/src/vespa/fastlib/text/.gitignore diff --git a/fastlib/src/vespa/fastlib/text/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/CMakeLists.txt index bcc3416331d..d6cb8c29305 100644 --- a/fastlib/src/vespa/fastlib/text/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/text/CMakeLists.txt @@ -1,9 +1,7 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_text +vespa_add_library(fastlib_text OBJECT SOURCES unicodeutil.cpp - wordfolder.cpp normwordfolder.cpp - INSTALL lib64 DEPENDS ) diff --git a/fastlib/src/vespa/fastlib/text/CustomProperties-4.0.0.txt b/vespalib/src/vespa/fastlib/text/CustomProperties-4.0.0.txt index 461d268cc6e..461d268cc6e 100644 --- a/fastlib/src/vespa/fastlib/text/CustomProperties-4.0.0.txt +++ b/vespalib/src/vespa/fastlib/text/CustomProperties-4.0.0.txt diff --git a/fastlib/src/vespa/fastlib/text/DerivedCoreProperties-4.0.0.txt b/vespalib/src/vespa/fastlib/text/DerivedCoreProperties-4.0.0.txt index 0fb45afd6af..0fb45afd6af 100644 --- a/fastlib/src/vespa/fastlib/text/DerivedCoreProperties-4.0.0.txt +++ b/vespalib/src/vespa/fastlib/text/DerivedCoreProperties-4.0.0.txt diff --git a/fastlib/src/vespa/fastlib/text/PropList-3.0.0.txt b/vespalib/src/vespa/fastlib/text/PropList-3.0.0.txt index 048efb2d369..048efb2d369 100644 --- a/fastlib/src/vespa/fastlib/text/PropList-3.0.0.txt +++ b/vespalib/src/vespa/fastlib/text/PropList-3.0.0.txt diff --git a/fastlib/src/vespa/fastlib/text/PropList-4.0.0.txt b/vespalib/src/vespa/fastlib/text/PropList-4.0.0.txt index 90176ad019b..90176ad019b 100644 --- a/fastlib/src/vespa/fastlib/text/PropList-4.0.0.txt +++ b/vespalib/src/vespa/fastlib/text/PropList-4.0.0.txt diff --git a/fastlib/src/vespa/fastlib/text/UCD-License b/vespalib/src/vespa/fastlib/text/UCD-License index 25cd5fa40e1..25cd5fa40e1 100644 --- a/fastlib/src/vespa/fastlib/text/UCD-License +++ b/vespalib/src/vespa/fastlib/text/UCD-License diff --git a/fastlib/src/vespa/fastlib/text/UnicodeData-3.0.0.txt b/vespalib/src/vespa/fastlib/text/UnicodeData-3.0.0.txt index 6a54d3d74e9..6a54d3d74e9 100644 --- a/fastlib/src/vespa/fastlib/text/UnicodeData-3.0.0.txt +++ b/vespalib/src/vespa/fastlib/text/UnicodeData-3.0.0.txt diff --git a/fastlib/src/vespa/fastlib/text/UnicodeData-4.0.0.txt b/vespalib/src/vespa/fastlib/text/UnicodeData-4.0.0.txt index 5394611560d..5394611560d 100644 --- a/fastlib/src/vespa/fastlib/text/UnicodeData-4.0.0.txt +++ b/vespalib/src/vespa/fastlib/text/UnicodeData-4.0.0.txt diff --git a/fastlib/src/vespa/fastlib/text/alphasort/AlphaSort1_0.dtd b/vespalib/src/vespa/fastlib/text/alphasort/AlphaSort1_0.dtd index 6c9a45bbbfc..6c9a45bbbfc 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/AlphaSort1_0.dtd +++ b/vespalib/src/vespa/fastlib/text/alphasort/AlphaSort1_0.dtd diff --git a/fastlib/src/vespa/fastlib/text/alphasort/AlphaSortMasterFile.xml b/vespalib/src/vespa/fastlib/text/alphasort/AlphaSortMasterFile.xml index 6a9da8cc1cf..6a9da8cc1cf 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/AlphaSortMasterFile.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/AlphaSortMasterFile.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Arabic.xml b/vespalib/src/vespa/fastlib/text/alphasort/Arabic.xml index 9789f519fd2..9789f519fd2 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Arabic.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Arabic.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Cyrillic.xml b/vespalib/src/vespa/fastlib/text/alphasort/Cyrillic.xml index 7332d1633ca..7332d1633ca 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Cyrillic.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Cyrillic.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Greek.xml b/vespalib/src/vespa/fastlib/text/alphasort/Greek.xml index 662003aefc6..662003aefc6 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Greek.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Greek.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hangul.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hangul.xml index 44ce855869e..44ce855869e 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hangul.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hangul.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-pinyin.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-pinyin.xml index 5f5486aaf4b..5f5486aaf4b 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-pinyin.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-pinyin.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-radical.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-radical.xml index f6b2a47642d..f6b2a47642d 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-radical.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-simplified-by-radical.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-pinyin.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-pinyin.xml index 93fe57de8d2..93fe57de8d2 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-pinyin.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-pinyin.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-radical.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-radical.xml index 28223fdcb2a..28223fdcb2a 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-radical.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-radical.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-stroke.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-stroke.xml index f9b4cceb89d..f9b4cceb89d 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-stroke.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hanzi-traditional-by-stroke.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Hebrew.xml b/vespalib/src/vespa/fastlib/text/alphasort/Hebrew.xml index 320aa1eacd7..320aa1eacd7 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Hebrew.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Hebrew.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Kana.xml b/vespalib/src/vespa/fastlib/text/alphasort/Kana.xml index 64d67a97719..64d67a97719 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Kana.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Kana.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Kanji-by-radical.xml b/vespalib/src/vespa/fastlib/text/alphasort/Kanji-by-radical.xml index 66b1fe4c447..66b1fe4c447 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Kanji-by-radical.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Kanji-by-radical.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Latin.xml b/vespalib/src/vespa/fastlib/text/alphasort/Latin.xml index 4c0df6aa186..4c0df6aa186 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Latin.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Latin.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Numbers.xml b/vespalib/src/vespa/fastlib/text/alphasort/Numbers.xml index e73b1fd2fd8..e73b1fd2fd8 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Numbers.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Numbers.xml diff --git a/fastlib/src/vespa/fastlib/text/alphasort/SortMaster1_0.dtd b/vespalib/src/vespa/fastlib/text/alphasort/SortMaster1_0.dtd index 9877d1bf8ca..9877d1bf8ca 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/SortMaster1_0.dtd +++ b/vespalib/src/vespa/fastlib/text/alphasort/SortMaster1_0.dtd diff --git a/fastlib/src/vespa/fastlib/text/alphasort/Space.xml b/vespalib/src/vespa/fastlib/text/alphasort/Space.xml index bb6b08c001a..bb6b08c001a 100644 --- a/fastlib/src/vespa/fastlib/text/alphasort/Space.xml +++ b/vespalib/src/vespa/fastlib/text/alphasort/Space.xml diff --git a/fastlib/src/vespa/fastlib/text/apps/.gitignore b/vespalib/src/vespa/fastlib/text/apps/.gitignore index c452e2714be..c452e2714be 100644 --- a/fastlib/src/vespa/fastlib/text/apps/.gitignore +++ b/vespalib/src/vespa/fastlib/text/apps/.gitignore diff --git a/fastlib/src/vespa/fastlib/text/apps/CMakeLists.txt b/vespalib/src/vespa/fastlib/text/apps/CMakeLists.txt index fd1049fa689..fd1049fa689 100644 --- a/fastlib/src/vespa/fastlib/text/apps/CMakeLists.txt +++ b/vespalib/src/vespa/fastlib/text/apps/CMakeLists.txt diff --git a/fastlib/src/vespa/fastlib/text/apps/extcase.cpp b/vespalib/src/vespa/fastlib/text/apps/extcase.cpp index 23486330f66..23486330f66 100644 --- a/fastlib/src/vespa/fastlib/text/apps/extcase.cpp +++ b/vespalib/src/vespa/fastlib/text/apps/extcase.cpp diff --git a/fastlib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp b/vespalib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp index 5f2b6089cdf..5f2b6089cdf 100644 --- a/fastlib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp +++ b/vespalib/src/vespa/fastlib/text/apps/unicode_propertydump.cpp diff --git a/fastlib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp b/vespalib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp index 7d84f6e931d..7d84f6e931d 100644 --- a/fastlib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp +++ b/vespalib/src/vespa/fastlib/text/apps/unicode_tolowerdump.cpp diff --git a/vespalib/src/vespa/fastlib/text/normwordfolder.cpp b/vespalib/src/vespa/fastlib/text/normwordfolder.cpp new file mode 100644 index 00000000000..a8454f42363 --- /dev/null +++ b/vespalib/src/vespa/fastlib/text/normwordfolder.cpp @@ -0,0 +1,599 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "normwordfolder.h" +#include <mutex> +#include <cstring> + +bool Fast_NormalizeWordFolder::_isInitialized = false; +std::mutex _initMutex; +bool Fast_NormalizeWordFolder::_doAccentRemoval = false; +bool Fast_NormalizeWordFolder::_doSharpSSubstitution = false; +bool Fast_NormalizeWordFolder::_doLigatureSubstitution = false; +bool Fast_NormalizeWordFolder::_doMulticharExpansion = false; +bool Fast_NormalizeWordFolder::_isWord[128]; + +ucs4_t Fast_NormalizeWordFolder::_foldCase[767]; // Up to Latin Extended B (0x0250) +ucs4_t Fast_NormalizeWordFolder::_foldCaseHighAscii[256]; // Latin Extended Additional (0x1E00 - 0x1F00) +ucs4_t Fast_NormalizeWordFolder::_kanaMap[192]; +ucs4_t Fast_NormalizeWordFolder::_halfwidth_fullwidthMap[240]; + +void +Fast_NormalizeWordFolder::Setup(uint32_t flags) +{ + // Only allow setting these when not initialized or initializing... + { + std::lock_guard<std::mutex> initGuard(_initMutex); + _doAccentRemoval = (DO_ACCENT_REMOVAL & flags) != 0; + _doSharpSSubstitution = (DO_SHARP_S_SUBSTITUTION & flags) != 0; + _doLigatureSubstitution = (DO_LIGATURE_SUBSTITUTION & flags) != 0; + _doMulticharExpansion = (DO_MULTICHAR_EXPANSION & flags) != 0; + _isInitialized = false; + } + Initialize(); +} + +void +Fast_NormalizeWordFolder::Initialize() +{ + unsigned int i; + if (!_isInitialized) { + std::lock_guard<std::mutex> initGuard(_initMutex); + if (!_isInitialized) { + + for (i = 0; i < 128; i++) + _isWord[i] = Fast_UnicodeUtil::IsWordChar(i); + for (i = 0; i < 767; i++) { + _foldCase[i] = Fast_UnicodeUtil::ToLower(i); + } + + for (i = 0x1E00; i < 0x1F00; i++) { + _foldCaseHighAscii[i - 0x1E00] = Fast_UnicodeUtil::ToLower(i); + } + + if (_doAccentRemoval) { + _foldCase[0xc0] = 'a'; + _foldCase[0xc1] = 'a'; + _foldCase[0xc2] = 'a'; + _foldCase[0xc3] = 'a'; // A tilde + _foldCase[0xc7] = 'c'; + _foldCase[0xc8] = 'e'; + _foldCase[0xc9] = 'e'; + _foldCase[0xca] = 'e'; + _foldCase[0xcb] = 'e'; + _foldCase[0xcc] = 'i'; // I grave + _foldCase[0xcd] = 'i'; + _foldCase[0xce] = 'i'; + _foldCase[0xcf] = 'i'; + _foldCase[0xd1] = 'n'; + _foldCase[0xd2] = 'o'; + _foldCase[0xd3] = 'o'; + _foldCase[0xd4] = 'o'; + _foldCase[0xd5] = 'o'; + _foldCase[0xd9] = 'u'; + _foldCase[0xda] = 'u'; + _foldCase[0xdb] = 'u'; + _foldCase[0xdd] = 'y'; + + _foldCase[0xe0] = 'a'; + _foldCase[0xe1] = 'a'; + _foldCase[0xe2] = 'a'; + _foldCase[0xe3] = 'a'; // a tilde + _foldCase[0xe7] = 'c'; + _foldCase[0xe8] = 'e'; + _foldCase[0xe9] = 'e'; + _foldCase[0xea] = 'e'; + _foldCase[0xeb] = 'e'; + _foldCase[0xec] = 'i'; // i grave + _foldCase[0xed] = 'i'; + _foldCase[0xee] = 'i'; + _foldCase[0xef] = 'i'; + _foldCase[0xf1] = 'n'; + _foldCase[0xf2] = 'o'; + _foldCase[0xf3] = 'o'; + _foldCase[0xf4] = 'o'; + _foldCase[0xf5] = 'o'; + _foldCase[0xf9] = 'u'; + _foldCase[0xfa] = 'u'; + _foldCase[0xfb] = 'u'; + _foldCase[0xfd] = 'y'; + _foldCase[0xff] = 'y'; + _foldCase[0x102] = 'a'; + _foldCase[0x103] = 'a'; + _foldCase[0x110] = 'd'; + _foldCase[0x111] = 'd'; + _foldCase[0x128] = 'i'; + _foldCase[0x129] = 'i'; + _foldCase[0x178] = 'y'; + _foldCase[0x1a0] = 'o'; + _foldCase[0x1a1] = 'o'; + _foldCase[0x1af] = 'u'; + _foldCase[0x1b0] = 'u'; + + // Superscript spacing modifiers + _foldCase[0x2b0] = 'h'; + _foldCase[0x2b1] = 0x266; + _foldCase[0x2b2] = 'j'; + _foldCase[0x2b3] = 'r'; + _foldCase[0x2b4] = 0x279; + _foldCase[0x2b5] = 0x27b; + _foldCase[0x2b6] = 0x281; + _foldCase[0x2b7] = 'w'; + _foldCase[0x2b8] = 'y'; + _foldCase[0x2e0] = 0x263; + _foldCase[0x2e1] = 'l'; + _foldCase[0x2e2] = 's'; + _foldCase[0x2e3] = 'x'; + _foldCase[0x2e4] = 0x295; + + // Superscript spacing modifiers + _foldCase[0x2b0] = 'h'; + _foldCase[0x2b1] = 0x266; + _foldCase[0x2b2] = 'j'; + _foldCase[0x2b3] = 'r'; + _foldCase[0x2b4] = 0x279; + _foldCase[0x2b5] = 0x27b; + _foldCase[0x2b6] = 0x281; + _foldCase[0x2b7] = 'w'; + _foldCase[0x2b8] = 'y'; + _foldCase[0x2e0] = 0x263; + _foldCase[0x2e1] = 'l'; + _foldCase[0x2e2] = 's'; + _foldCase[0x2e3] = 'x'; + _foldCase[0x2e4] = 0x295; + + // Deaccenting-table for Ascii Extended Additional + _foldCaseHighAscii[0x1ea0 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea1 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea2 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea3 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea4 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea5 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea6 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea7 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea8 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ea9 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eaa - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eab - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eac - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1ead - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eae - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eaf - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb0 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb1 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb2 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb3 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb4 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb5 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb6 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb7 - 0x1e00] = 'a'; + _foldCaseHighAscii[0x1eb8 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1eb9 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1eba - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ebb - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ebc - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ebd - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ebe - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ebf - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec0 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec1 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec2 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec3 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec4 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec5 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec6 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec7 - 0x1e00] = 'e'; + _foldCaseHighAscii[0x1ec8 - 0x1e00] = 'i'; + _foldCaseHighAscii[0x1ec9 - 0x1e00] = 'i'; + _foldCaseHighAscii[0x1eca - 0x1e00] = 'i'; + _foldCaseHighAscii[0x1ecb - 0x1e00] = 'i'; + _foldCaseHighAscii[0x1ecc - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ecd - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ece - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ecf - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed0 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed1 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed2 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed3 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed4 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed5 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed6 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed7 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed8 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ed9 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1eda - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1edb - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1edc - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1edd - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ede - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1edf - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ee0 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ee1 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ee2 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ee3 - 0x1e00] = 'o'; + _foldCaseHighAscii[0x1ee4 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ee5 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ee6 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ee7 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ee8 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ee9 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1eea - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1eeb - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1eec - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1eed - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1eee - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1eef - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ef0 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ef1 - 0x1e00] = 'u'; + _foldCaseHighAscii[0x1ef2 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef3 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef4 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef5 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef6 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef7 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef8 - 0x1e00] = 'y'; + _foldCaseHighAscii[0x1ef9 - 0x1e00] = 'y'; + } + + // Base case hiragana - hiragana ID + for (i = 0; i < 96; i++) { + _kanaMap[i] = 0x3040 + i; + } + + // katakana - katakana ID + for (i = 96; i < 192; i++) { + _kanaMap[i] = 0x3040 + i; + } + + // Fullwidth ASCII + for (i = 0; i < 0x21; i++) + _halfwidth_fullwidthMap[i] = 0x20 + i; + for (i = 0x21; i < 0x3B; i++) // full uppercase to half lowercase + _halfwidth_fullwidthMap[i] = 0x40 + i; + for (i = 0x3B; i < 0x5F; i++) + _halfwidth_fullwidthMap[i] = 0x20 + i; + // 0xFF00, 0xFF5F -> id + _halfwidth_fullwidthMap[0x00] = 0xFF00; + _halfwidth_fullwidthMap[0x5F] = 0xFF5F; + + // Halfwidth CJK Punctuation + // 0xFF60 -> id + _halfwidth_fullwidthMap[0x60] = 0xFF60; + _halfwidth_fullwidthMap[0x61] = 0x3002; + _halfwidth_fullwidthMap[0x62] = 0x300C; + _halfwidth_fullwidthMap[0x63] = 0x300D; + _halfwidth_fullwidthMap[0x64] = 0x3001; + + // Halfwidth katakana (maps directly to hiragana) + + // Common cases for halfwidth katakana + _halfwidth_fullwidthMap[0x65] = 0x30FB; + + _halfwidth_fullwidthMap[0x66] = 0x30F2; + _halfwidth_fullwidthMap[0x6F] = 0x30C3; + _halfwidth_fullwidthMap[0x70] = 0x30FC; + _halfwidth_fullwidthMap[0x71] = 0x30A2; + _halfwidth_fullwidthMap[0x72] = 0x30A4; + _halfwidth_fullwidthMap[0x73] = 0x30A6; + _halfwidth_fullwidthMap[0x74] = 0x30A8; + _halfwidth_fullwidthMap[0x75] = 0x30AA; + _halfwidth_fullwidthMap[0x76] = 0x30AB; + _halfwidth_fullwidthMap[0x77] = 0x30AD; + _halfwidth_fullwidthMap[0x78] = 0x30AF; + _halfwidth_fullwidthMap[0x79] = 0x30B1; + _halfwidth_fullwidthMap[0x7A] = 0x30B3; + _halfwidth_fullwidthMap[0x7B] = 0x30B5; + _halfwidth_fullwidthMap[0x7C] = 0x30B7; + _halfwidth_fullwidthMap[0x7D] = 0x30B9; + _halfwidth_fullwidthMap[0x7E] = 0x30BB; + _halfwidth_fullwidthMap[0x7F] = 0x30BD; + _halfwidth_fullwidthMap[0x80] = 0x30BF; + _halfwidth_fullwidthMap[0x81] = 0x30C1; + _halfwidth_fullwidthMap[0x82] = 0x30C4; + _halfwidth_fullwidthMap[0x83] = 0x30C6; + _halfwidth_fullwidthMap[0x84] = 0x30C8; + _halfwidth_fullwidthMap[0x85] = 0x30CA; + _halfwidth_fullwidthMap[0x86] = 0x30CB; + _halfwidth_fullwidthMap[0x87] = 0x30CC; + _halfwidth_fullwidthMap[0x88] = 0x30CD; + _halfwidth_fullwidthMap[0x89] = 0x30CE; + _halfwidth_fullwidthMap[0x8A] = 0x30CF; + _halfwidth_fullwidthMap[0x8B] = 0x30D2; + _halfwidth_fullwidthMap[0x8C] = 0x30D5; + _halfwidth_fullwidthMap[0x8D] = 0x30D8; + _halfwidth_fullwidthMap[0x8E] = 0x30DB; + _halfwidth_fullwidthMap[0x8F] = 0x30DE; + _halfwidth_fullwidthMap[0x90] = 0x30DF; + _halfwidth_fullwidthMap[0x91] = 0x30E0; + _halfwidth_fullwidthMap[0x92] = 0x30E1; + _halfwidth_fullwidthMap[0x93] = 0x30E2; + _halfwidth_fullwidthMap[0x94] = 0x30E4; + _halfwidth_fullwidthMap[0x95] = 0x30E6; + _halfwidth_fullwidthMap[0x96] = 0x30E8; + _halfwidth_fullwidthMap[0x97] = 0x30E9; + _halfwidth_fullwidthMap[0x98] = 0x30EA; + _halfwidth_fullwidthMap[0x99] = 0x30EB; + _halfwidth_fullwidthMap[0x9A] = 0x30EC; + _halfwidth_fullwidthMap[0x9B] = 0x30ED; + _halfwidth_fullwidthMap[0x9C] = 0x30EF; + _halfwidth_fullwidthMap[0x9D] = 0x30F3; + _halfwidth_fullwidthMap[0x9E] = 0x3099; + _halfwidth_fullwidthMap[0x9F] = 0x309A; + + _halfwidth_fullwidthMap[0x67] = 0x30a1; + _halfwidth_fullwidthMap[0x68] = 0x30a3; + _halfwidth_fullwidthMap[0x69] = 0x30a5; + _halfwidth_fullwidthMap[0x6A] = 0x30a7; + _halfwidth_fullwidthMap[0x6B] = 0x30a9; + _halfwidth_fullwidthMap[0x6C] = 0x30e3; + _halfwidth_fullwidthMap[0x6D] = 0x30e5; + _halfwidth_fullwidthMap[0x6E] = 0x30e7; + + // Halfwidth Hangul + _halfwidth_fullwidthMap[0xA0] = 0x3164; + // fill in 0xFFA1 - 0xFFBE => 0x3131 - 0x314E + for (i = 0xA1; i < 0xBF; i++) + _halfwidth_fullwidthMap[i] = 0x3090 + i; + _halfwidth_fullwidthMap[0xBF] = 0xFFBF; + _halfwidth_fullwidthMap[0xC0] = 0xFFC0; + _halfwidth_fullwidthMap[0xC1] = 0xFFC1; + // fill in 0xFFC2 - 0xFFC7 => 0x314F - 0x3154 + for (i = 0xC2; i < 0xC8; i++) + _halfwidth_fullwidthMap[i] = 0x308D + i; + _halfwidth_fullwidthMap[0xC8] = 0xFFC8; + _halfwidth_fullwidthMap[0xC9] = 0xFFC9; + // fill in 0xFFCA - 0xFFCF => 0x3155 - 0x315A + for (i = 0xCA; i < 0xD0; i++) + _halfwidth_fullwidthMap[i] = 0x308B + i; + _halfwidth_fullwidthMap[0xD0] = 0xFFD0; + _halfwidth_fullwidthMap[0xD1] = 0xFFD1; + // fill in 0xFFD2 - 0xFFD7 => 0x315B - 0x3160 + for (i = 0xD2; i < 0xD8; i++) + _halfwidth_fullwidthMap[i] = 0x3089 + i; + _halfwidth_fullwidthMap[0xD8] = 0xFFD8; + _halfwidth_fullwidthMap[0xD9] = 0xFFD9; + // fill in 0xFFDA - 0xFFDC => 0x3161 - 0x3163 + for (i = 0xDA; i < 0xDD; i++) + _halfwidth_fullwidthMap[i] = 0x3087 + i; + + // Fullwidth symbols + _halfwidth_fullwidthMap[0xE0] = 0x00A2; + _halfwidth_fullwidthMap[0xE1] = 0x00A3; + _halfwidth_fullwidthMap[0xE2] = 0x00AC; + _halfwidth_fullwidthMap[0xE3] = 0x00AF; + _halfwidth_fullwidthMap[0xE4] = 0x00A6; + _halfwidth_fullwidthMap[0xE5] = 0x00A5; + _halfwidth_fullwidthMap[0xE6] = 0x20A9; + + // 0xFFE7 -> id + _halfwidth_fullwidthMap[0xE7] = 0xFFE7; + + // Halfwidth symbols + _halfwidth_fullwidthMap[0xE8] = 0x2502; + _halfwidth_fullwidthMap[0xE9] = 0x2190; + _halfwidth_fullwidthMap[0xEA] = 0x2191; + _halfwidth_fullwidthMap[0xEB] = 0x2192; + _halfwidth_fullwidthMap[0xEC] = 0x2193; + _halfwidth_fullwidthMap[0xED] = 0x25A0; + _halfwidth_fullwidthMap[0xEE] = 0x25CB; + + // 0xFFEF -> id + _halfwidth_fullwidthMap[0xEF] = 0xFFEF; + + + // + // DONE + // + _isInitialized = true; + } + } +} + +Fast_NormalizeWordFolder::Fast_NormalizeWordFolder() +{ + Initialize(); +} + + +Fast_NormalizeWordFolder::~Fast_NormalizeWordFolder(void) +{ +} + +const char* +Fast_NormalizeWordFolder::UCS4Tokenize(const char *buf, + const char *bufend, + ucs4_t *dstbuf, + ucs4_t *dstbufend, + const char*& origstart, + size_t& tokenlen) const +{ + + ucs4_t c; + const unsigned char *p, *ep; + ucs4_t *q, *eq; + p = reinterpret_cast<const unsigned char *>(buf); + ep = reinterpret_cast<const unsigned char *>(bufend); + + // Skip characters between words + for (;;) { + if (p >= ep) { // End of input buffer, no more words + *dstbuf = 0; + return reinterpret_cast<const char *>(p); + } + if (*p < 128) { // Common case, ASCII + c = *p++; + if (_isWord[c]) + { + origstart = reinterpret_cast<const char *>(p) - 1; + break; + } + } else { + const unsigned char* prev_p = p; + c = Fast_UnicodeUtil::GetUTF8Char(p); + if (IsWordCharOrIA(c)) + { + origstart = reinterpret_cast<const char *>(prev_p); + break; + } + } + } + + // Start saving word. + q = dstbuf; + eq = dstbufend - 3; // Make room for UCS4 char replacement string and NUL + // Doesn't check for space for the first char, assumes that + // word buffer is at least 13 characters + if (c < 128) { // Common case, ASCII + *q++ = _foldCase[c]; + } else { + const char *repl = ReplacementString(c); + if (repl != nullptr) { + size_t repllen = strlen(repl); + if (repllen > 0) + q = Fast_UnicodeUtil::ucs4copy(q,repl); + } else { + c = ToFold(c); + *q++ = c; + } + } + + // Special case for interlinear annotation + if (c == 0xFFF9) { // ANCHOR + // Collect up to and including terminator + for(;;) { + if (p >= ep) { + c = 0; + break; + } + if (*p < 128) { // Note, no exit on plain ASCII + c = *p++; + *q++ = c; + if (q >= eq) { // Junk rest of annotation block + for (;;) { + if (p >= ep) { // End of input buffer + c = 0; + break; + } + if (*p < 128) { // Common case, ASCII + c = *p++; + } else { + c = Fast_UnicodeUtil::GetUTF8Char(p); + if (c == 0xFFFB) { + break; // out of junking loop + } + } + } + break; // out of annotation block processing + } + } else { + c = Fast_UnicodeUtil::GetUTF8Char(p); + *q++ = c; + if (c == 0xFFFB) { // TERMINATOR => Exit condition + break; + } + if (q >= eq) { // Junk rest of word + for (;;) { + if (p >= ep) { // End of input buffer + c = 0; + break; + } + if (*p < 128) { // Common case, ASCII + c = *p++; + } else { + c = Fast_UnicodeUtil::GetUTF8Char(p); + if (c == 0xFFFB) { + break; + } + } + } + break; + } + } + } + } else + + for (;;) { + if (p >= ep) { // End of input buffer + c = 0; + break; + } + if (*p < 128) { // Common case, ASCII + c = *p++; + if (!_isWord[c]) + { + p--; + break; + } + *q++ = _foldCase[c]; + if (q >= eq) { // Junk rest of word + for (;;) { + if (p >= ep) { // End of input buffer + c = 0; + break; + } + if (*p < 128) { // Common case, ASCII + c = *p++; + if (!_isWord[c]) + { + p--; + break; + } + } else { + const unsigned char* prev_p = p; + c = Fast_UnicodeUtil::GetUTF8Char(p); + if (!Fast_UnicodeUtil::IsWordChar(c)) + { + p = prev_p; + break; + } + } + } + break; + } + } else { + const unsigned char* prev_p = p; + c = Fast_UnicodeUtil::GetUTF8Char(p); + if (!Fast_UnicodeUtil::IsWordChar(c)) + { + p = prev_p; + break; + } + const char *repl = ReplacementString(c); + if (repl != nullptr) { + size_t repllen = strlen(repl); + if (repllen > 0) + q = Fast_UnicodeUtil::ucs4copy(q,repl); + } else { + c = ToFold(c); + *q++ = c; + } + if (q >= eq) { // Junk rest of word + for (;;) { + if (p >= ep) { // End of input buffer + c = 0; + break; + } + if (*p < 128) { // Common case, ASCII + c = *p++; + if (!_isWord[c]) + { + p--; + break; + } + } else { + const unsigned char* xprev_p = p; + c = Fast_UnicodeUtil::GetUTF8Char(p); + if (!Fast_UnicodeUtil::IsWordChar(c)) + { + p = xprev_p; + break; + } + } + } + break; + } + } + } + *q = 0; + tokenlen = q - dstbuf; + return reinterpret_cast<const char *>(p); +} diff --git a/fastlib/src/vespa/fastlib/text/normwordfolder.h b/vespalib/src/vespa/fastlib/text/normwordfolder.h index 7136a8b3829..dc345e7d9be 100644 --- a/fastlib/src/vespa/fastlib/text/normwordfolder.h +++ b/vespalib/src/vespa/fastlib/text/normwordfolder.h @@ -16,10 +16,6 @@ private: /** Features */ static bool _doAccentRemoval; - static bool _doSmallToNormalKana; - static bool _doKatakanaToHiragana; - static bool _doKanaAccentCollapsing; - static bool _doFullwidthToBasicLatin; static bool _doSharpSSubstitution; static bool _doLigatureSubstitution; static bool _doMulticharExpansion; @@ -57,8 +53,6 @@ public: static ucs4_t _foldCase[767]; // Up to Spacing Modifiers, inclusize (0x02FF) static ucs4_t _foldCaseHighAscii[256]; // Latin Extended Additional (0x1E00 - 0x1F00) (incl. vietnamese) private: - static ucs4_t _keepCase[767]; - static ucs4_t _keepCaseHighAscii[256]; /** Map the values from range 0x3040 (0) - 0x30FF (191). */ static ucs4_t _kanaMap[192]; static ucs4_t _halfwidth_fullwidthMap[240]; @@ -165,16 +159,9 @@ private: return Fast_UnicodeUtil::IsWordChar(c) || c == 0xFFF9 || c == 0xFFFA || c == 0xFFFB; } - public: Fast_NormalizeWordFolder(); - virtual ~Fast_NormalizeWordFolder(); - virtual size_t FoldedSizeAsUTF8(const char *word) const; - virtual char *FoldUTF8WordToUTF8Quick(char *wordbufpos, const char *word) const; - virtual const char *Tokenize(const char *buf, const char *bufend, char *dstbuf, - char *dstbufend, const char*& origstart, size_t& tokenlen) const; - virtual const char *Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, - ucs4_t *dstbufend, const char*& origstart, size_t& tokenlen) const; + ~Fast_NormalizeWordFolder() override; const char* UCS4Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, ucs4_t *dstbufend, const char*& origstart, size_t& tokenlen) const override; }; diff --git a/fastlib/src/vespa/fastlib/text/unicodeutil-charprops.cpp b/vespalib/src/vespa/fastlib/text/unicodeutil-charprops.cpp index ee59c5c84dd..ee59c5c84dd 100644 --- a/fastlib/src/vespa/fastlib/text/unicodeutil-charprops.cpp +++ b/vespalib/src/vespa/fastlib/text/unicodeutil-charprops.cpp diff --git a/fastlib/src/vespa/fastlib/text/unicodeutil-lowercase.cpp b/vespalib/src/vespa/fastlib/text/unicodeutil-lowercase.cpp index def65d642cd..def65d642cd 100644 --- a/fastlib/src/vespa/fastlib/text/unicodeutil-lowercase.cpp +++ b/vespalib/src/vespa/fastlib/text/unicodeutil-lowercase.cpp diff --git a/fastlib/src/vespa/fastlib/text/unicodeutil.cpp b/vespalib/src/vespa/fastlib/text/unicodeutil.cpp index 54ccce9f8e8..08107337ee4 100644 --- a/fastlib/src/vespa/fastlib/text/unicodeutil.cpp +++ b/vespalib/src/vespa/fastlib/text/unicodeutil.cpp @@ -2,14 +2,10 @@ #include "unicodeutil.h" #include <cstdlib> -#include <cstdint> -#include <cassert> #include "unicodeutil-charprops.cpp" #include "unicodeutil-lowercase.cpp" -unsigned char Fast_UnicodeUtil::_utf8header[256]; - namespace { class Initialize @@ -30,16 +26,6 @@ Fast_UnicodeUtil::InitTables() */ _compCharProps[(0xFF9E >> 8)][(0xFF9E & 255)] |= 32; _compCharProps[(0xFF9F >> 8)][(0xFF9F & 255)] |= 32; - - for (uint32_t i = 0; i < 256; i++) { _utf8header[i] = 0; } - - // Initialize _utf8header array - for (uint32_t i = 0x00; i <= 0x7F; i++) { _utf8header[i] = 1; } - for (uint32_t i = 0xC0; i <= 0xDF; i++) { _utf8header[i] = 2; } - for (uint32_t i = 0xE0; i <= 0xEF; i++) { _utf8header[i] = 3; } - for (uint32_t i = 0xF0; i <= 0xF7; i++) { _utf8header[i] = 4; } - for (uint32_t i = 0xF8; i <= 0xFB; i++) { _utf8header[i] = 5; } - for (uint32_t i = 0xFC; i <= 0xFD; i++) { _utf8header[i] = 6; } } char * @@ -140,31 +126,6 @@ Fast_UnicodeUtil::ucs4copy(ucs4_t *dst, const char *src) return p; } -char * -Fast_UnicodeUtil::strdupLAT1(const char *src) -{ - char *res; - size_t reslen; - ucs4_t i; - const unsigned char *p; - char *q; - - reslen = 0; - p = reinterpret_cast<const unsigned char *>(src); - while ((i = *p++) != 0) { - reslen += utf8clen(i); - } - res = static_cast<char *>(malloc(reslen + 1)); - p = reinterpret_cast<const unsigned char *>(src); - q = res; - while ((i = *p++) != 0) { - q = utf8cput(q, i); - } - assert(q == res + reslen); - *q = 0; - return res; -} - ucs4_t Fast_UnicodeUtil::GetUTF8CharNonAscii(unsigned const char *&src) { diff --git a/fastlib/src/vespa/fastlib/text/unicodeutil.h b/vespalib/src/vespa/fastlib/text/unicodeutil.h index d6ba0ae0e23..e155af134fb 100644 --- a/fastlib/src/vespa/fastlib/text/unicodeutil.h +++ b/vespalib/src/vespa/fastlib/text/unicodeutil.h @@ -6,11 +6,6 @@ #include <sys/types.h> -#define FASTLIB_UNICODEUTIL_USES_WORDCHARPROP - - -/** utf8_t is the type of the multi-byte UTF-8 character components */ -typedef unsigned char utf8_t; /** ucs4_t is the type of the 4-byte UCS4 characters */ typedef unsigned int ucs4_t; @@ -22,10 +17,6 @@ typedef unsigned int ucs4_t; class Fast_UnicodeUtil { private: /** - * Table for easy lookup of UTF8 character length in bytes - */ - static unsigned char _utf8header[256]; - /** * Is true when the tables have been initialized. Is set by * InitTables, and should be protected by the _initMutex before * inspection. @@ -220,14 +211,6 @@ public: static int utf8cmp(const char *s1, const ucs4_t *s2); /** - * Copy an ISO-8859-1 string to an UTF-8 string. - * @param src The source ISO-8859-1 string. - * @return Pointer to a new alloacted buffer with the UTF-8 result. - * NB Only use in local test - */ - static char *strdupLAT1(const char *src); - - /** * Test for terminal punctuation. * @param testchar the UCS4 character to test. * @return true if testchar is a terminal punctuation character, diff --git a/fastlib/src/vespa/fastlib/text/wordfolder.h b/vespalib/src/vespa/fastlib/text/wordfolder.h index b0edd2d1385..8b75606f828 100644 --- a/fastlib/src/vespa/fastlib/text/wordfolder.h +++ b/vespalib/src/vespa/fastlib/text/wordfolder.h @@ -6,7 +6,7 @@ class Fast_WordFolder { public: - virtual ~Fast_WordFolder(void); + virtual ~Fast_WordFolder() = default; virtual const char* UCS4Tokenize(const char *buf, const char *bufend, ucs4_t *dstbuf, diff --git a/vespalib/src/vespa/vespalib/CMakeLists.txt b/vespalib/src/vespa/vespalib/CMakeLists.txt index 1fffddf2f2f..024d61b89e5 100644 --- a/vespalib/src/vespa/vespalib/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/CMakeLists.txt @@ -7,12 +7,15 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_data> $<TARGET_OBJECTS:vespalib_vespalib_data_slime> $<TARGET_OBJECTS:vespalib_vespalib_datastore> + $<TARGET_OBJECTS:vespalib_vespalib_encoding> $<TARGET_OBJECTS:vespalib_vespalib_fuzzy> $<TARGET_OBJECTS:vespalib_vespalib_geo> $<TARGET_OBJECTS:vespalib_vespalib_hwaccelrated> $<TARGET_OBJECTS:vespalib_vespalib_io> $<TARGET_OBJECTS:vespalib_vespalib_locale> + $<TARGET_OBJECTS:vespalib_vespalib_metrics> $<TARGET_OBJECTS:vespalib_vespalib_net> + $<TARGET_OBJECTS:vespalib_vespalib_net_http> $<TARGET_OBJECTS:vespalib_vespalib_net_tls> $<TARGET_OBJECTS:vespalib_vespalib_net_tls_impl> $<TARGET_OBJECTS:vespalib_vespalib_objects> @@ -26,6 +29,8 @@ vespa_add_library(vespalib $<TARGET_OBJECTS:vespalib_vespalib_time> $<TARGET_OBJECTS:vespalib_vespalib_trace> $<TARGET_OBJECTS:vespalib_vespalib_util> + $<TARGET_OBJECTS:fastlib_io> + $<TARGET_OBJECTS:fastlib_text> INSTALL lib64 DEPENDS ${VESPA_GCC_LIB} diff --git a/vespalib/src/vespa/vespalib/data/CMakeLists.txt b/vespalib/src/vespa/vespalib/data/CMakeLists.txt index 39ff0661c1f..ff7cdacb04f 100644 --- a/vespalib/src/vespa/vespalib/data/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/data/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(vespalib_vespalib_data OBJECT SOURCES databuffer.cpp + fileheader.cpp input.cpp input_reader.cpp lz4_input_decoder.cpp diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp b/vespalib/src/vespa/vespalib/data/fileheader.cpp index 0cb5fa14ff4..0cb5fa14ff4 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.cpp +++ b/vespalib/src/vespa/vespalib/data/fileheader.cpp diff --git a/staging_vespalib/src/vespa/vespalib/data/fileheader.h b/vespalib/src/vespa/vespalib/data/fileheader.h index ca475971932..ca475971932 100644 --- a/staging_vespalib/src/vespa/vespalib/data/fileheader.h +++ b/vespalib/src/vespa/vespalib/data/fileheader.h diff --git a/staging_vespalib/src/vespa/vespalib/encoding/.gitignore b/vespalib/src/vespa/vespalib/encoding/.gitignore index ee8938b6bf4..ee8938b6bf4 100644 --- a/staging_vespalib/src/vespa/vespalib/encoding/.gitignore +++ b/vespalib/src/vespa/vespalib/encoding/.gitignore diff --git a/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt b/vespalib/src/vespa/vespalib/encoding/CMakeLists.txt index 48fed5ae1cc..604930d330e 100644 --- a/staging_vespalib/src/vespa/vespalib/encoding/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/encoding/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_encoding OBJECT +vespa_add_library(vespalib_vespalib_encoding OBJECT SOURCES base64.cpp DEPENDS diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp b/vespalib/src/vespa/vespalib/encoding/base64.cpp index bad3c168b15..bad3c168b15 100644 --- a/staging_vespalib/src/vespa/vespalib/encoding/base64.cpp +++ b/vespalib/src/vespa/vespalib/encoding/base64.cpp diff --git a/staging_vespalib/src/vespa/vespalib/encoding/base64.h b/vespalib/src/vespa/vespalib/encoding/base64.h index e54ae5765d9..e54ae5765d9 100644 --- a/staging_vespalib/src/vespa/vespalib/encoding/base64.h +++ b/vespalib/src/vespa/vespalib/encoding/base64.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt b/vespalib/src/vespa/vespalib/metrics/CMakeLists.txt index efe9e848136..b29f5d0bbe9 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/metrics/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_metrics OBJECT +vespa_add_library(vespalib_vespalib_metrics OBJECT SOURCES bucket.cpp clock.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp b/vespalib/src/vespa/vespalib/metrics/bucket.cpp index 8d4d5558c3d..8d4d5558c3d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/bucket.cpp +++ b/vespalib/src/vespa/vespalib/metrics/bucket.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/bucket.h b/vespalib/src/vespa/vespalib/metrics/bucket.h index 75387aad1b3..75387aad1b3 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/bucket.h +++ b/vespalib/src/vespa/vespalib/metrics/bucket.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp b/vespalib/src/vespa/vespalib/metrics/clock.cpp index 8593e07998e..8593e07998e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.cpp +++ b/vespalib/src/vespa/vespalib/metrics/clock.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/clock.h b/vespalib/src/vespa/vespalib/metrics/clock.h index e1b0a7da003..e1b0a7da003 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/clock.h +++ b/vespalib/src/vespa/vespalib/metrics/clock.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp b/vespalib/src/vespa/vespalib/metrics/counter.cpp index 2b94ffce842..2b94ffce842 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter.cpp +++ b/vespalib/src/vespa/vespalib/metrics/counter.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter.h b/vespalib/src/vespa/vespalib/metrics/counter.h index 751c2cc3806..751c2cc3806 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter.h +++ b/vespalib/src/vespa/vespalib/metrics/counter.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp b/vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp index 583bb6e9a1c..583bb6e9a1c 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp +++ b/vespalib/src/vespa/vespalib/metrics/counter_aggregator.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.h b/vespalib/src/vespa/vespalib/metrics/counter_aggregator.h index c650b67dad3..c650b67dad3 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/counter_aggregator.h +++ b/vespalib/src/vespa/vespalib/metrics/counter_aggregator.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.cpp b/vespalib/src/vespa/vespalib/metrics/current_samples.cpp index 67d56d6f748..67d56d6f748 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.cpp +++ b/vespalib/src/vespa/vespalib/metrics/current_samples.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.h b/vespalib/src/vespa/vespalib/metrics/current_samples.h index 4056a4bb6aa..4056a4bb6aa 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/current_samples.h +++ b/vespalib/src/vespa/vespalib/metrics/current_samples.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp b/vespalib/src/vespa/vespalib/metrics/dimension.cpp index bea751a6680..bea751a6680 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dimension.cpp +++ b/vespalib/src/vespa/vespalib/metrics/dimension.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h b/vespalib/src/vespa/vespalib/metrics/dimension.h index 7a3942b705c..7a3942b705c 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dimension.h +++ b/vespalib/src/vespa/vespalib/metrics/dimension.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp b/vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp index 13cab477b2e..13cab477b2e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp +++ b/vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h b/vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h index a707c3a5305..a707c3a5305 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h +++ b/vespalib/src/vespa/vespalib/metrics/dummy_metrics_manager.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp b/vespalib/src/vespa/vespalib/metrics/gauge.cpp index ca6b11697b4..ca6b11697b4 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge.cpp +++ b/vespalib/src/vespa/vespalib/metrics/gauge.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h b/vespalib/src/vespa/vespalib/metrics/gauge.h index 59de4a6fe4d..59de4a6fe4d 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge.h +++ b/vespalib/src/vespa/vespalib/metrics/gauge.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp b/vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp index a6c4559931f..a6c4559931f 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp +++ b/vespalib/src/vespa/vespalib/metrics/gauge_aggregator.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h b/vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h index f4fd4760cf1..f4fd4760cf1 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h +++ b/vespalib/src/vespa/vespalib/metrics/gauge_aggregator.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/handle.cpp b/vespalib/src/vespa/vespalib/metrics/handle.cpp index 50143adedd2..50143adedd2 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/handle.cpp +++ b/vespalib/src/vespa/vespalib/metrics/handle.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/handle.h b/vespalib/src/vespa/vespalib/metrics/handle.h index b901f56af7e..b901f56af7e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/handle.h +++ b/vespalib/src/vespa/vespalib/metrics/handle.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp b/vespalib/src/vespa/vespalib/metrics/json_formatter.cpp index dddc5781458..dddc5781458 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.cpp +++ b/vespalib/src/vespa/vespalib/metrics/json_formatter.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h b/vespalib/src/vespa/vespalib/metrics/json_formatter.h index f9d3664e5db..f9d3664e5db 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/json_formatter.h +++ b/vespalib/src/vespa/vespalib/metrics/json_formatter.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.cpp b/vespalib/src/vespa/vespalib/metrics/label.cpp index fe287e16d46..fe287e16d46 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/label.cpp +++ b/vespalib/src/vespa/vespalib/metrics/label.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/label.h b/vespalib/src/vespa/vespalib/metrics/label.h index 755b3e83bef..755b3e83bef 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/label.h +++ b/vespalib/src/vespa/vespalib/metrics/label.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.cpp b/vespalib/src/vespa/vespalib/metrics/metric_id.cpp index 7ce012ad5fd..7ce012ad5fd 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.cpp +++ b/vespalib/src/vespa/vespalib/metrics/metric_id.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.h b/vespalib/src/vespa/vespalib/metrics/metric_id.h index 5aa9d1b868e..5aa9d1b868e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_id.h +++ b/vespalib/src/vespa/vespalib/metrics/metric_id.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp b/vespalib/src/vespa/vespalib/metrics/metric_types.cpp index 196dfaed16a..196dfaed16a 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.cpp +++ b/vespalib/src/vespa/vespalib/metrics/metric_types.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h b/vespalib/src/vespa/vespalib/metrics/metric_types.h index e8118b6bd63..e8118b6bd63 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metric_types.h +++ b/vespalib/src/vespa/vespalib/metrics/metric_types.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp b/vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp index 3d598528777..3d598528777 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp +++ b/vespalib/src/vespa/vespalib/metrics/metrics_manager.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h b/vespalib/src/vespa/vespalib/metrics/metrics_manager.h index 6b80527b22e..6b80527b22e 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/metrics_manager.h +++ b/vespalib/src/vespa/vespalib/metrics/metrics_manager.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp b/vespalib/src/vespa/vespalib/metrics/name_collection.cpp index 964fd9c59a8..964fd9c59a8 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.cpp +++ b/vespalib/src/vespa/vespalib/metrics/name_collection.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h b/vespalib/src/vespa/vespalib/metrics/name_collection.h index bef614b5a68..bef614b5a68 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_collection.h +++ b/vespalib/src/vespa/vespalib/metrics/name_collection.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.cpp b/vespalib/src/vespa/vespalib/metrics/name_repo.cpp index 2f31c90da29..2f31c90da29 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.cpp +++ b/vespalib/src/vespa/vespalib/metrics/name_repo.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.h b/vespalib/src/vespa/vespalib/metrics/name_repo.h index e2230cf59d9..e2230cf59d9 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/name_repo.h +++ b/vespalib/src/vespa/vespalib/metrics/name_repo.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.cpp b/vespalib/src/vespa/vespalib/metrics/point.cpp index b4d3d6e2738..b4d3d6e2738 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point.cpp +++ b/vespalib/src/vespa/vespalib/metrics/point.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point.h b/vespalib/src/vespa/vespalib/metrics/point.h index a8f483c4404..a8f483c4404 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point.h +++ b/vespalib/src/vespa/vespalib/metrics/point.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp b/vespalib/src/vespa/vespalib/metrics/point_builder.cpp index d085228c1a8..d085228c1a8 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.cpp +++ b/vespalib/src/vespa/vespalib/metrics/point_builder.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h b/vespalib/src/vespa/vespalib/metrics/point_builder.h index d9c4b5a114f..d9c4b5a114f 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_builder.h +++ b/vespalib/src/vespa/vespalib/metrics/point_builder.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp b/vespalib/src/vespa/vespalib/metrics/point_map.cpp index cd5909707e3..cd5909707e3 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map.cpp +++ b/vespalib/src/vespa/vespalib/metrics/point_map.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h b/vespalib/src/vespa/vespalib/metrics/point_map.h index 7a435389566..7a435389566 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map.h +++ b/vespalib/src/vespa/vespalib/metrics/point_map.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp b/vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp index d9b7362521b..d9b7362521b 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp +++ b/vespalib/src/vespa/vespalib/metrics/point_map_collection.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h b/vespalib/src/vespa/vespalib/metrics/point_map_collection.h index ee556ea2107..ee556ea2107 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/point_map_collection.h +++ b/vespalib/src/vespa/vespalib/metrics/point_map_collection.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp b/vespalib/src/vespa/vespalib/metrics/producer.cpp index de3ed5d368f..de3ed5d368f 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/producer.cpp +++ b/vespalib/src/vespa/vespalib/metrics/producer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/producer.h b/vespalib/src/vespa/vespalib/metrics/producer.h index ac741b4e82d..397b0979b25 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/producer.h +++ b/vespalib/src/vespa/vespalib/metrics/producer.h @@ -1,11 +1,10 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/vespalib/net/http/metrics_producer.h> #include <memory> -#include <vespa/vespalib/net/metrics_producer.h> -namespace vespalib { -namespace metrics { +namespace vespalib::metrics { class MetricsManager; @@ -21,5 +20,4 @@ public: vespalib::string getTotalMetrics(const vespalib::string &consumer) override; }; -} // namespace vespalib::metrics -} // namespace vespalib +} diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp b/vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp index d438426518c..d438426518c 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp +++ b/vespalib/src/vespa/vespalib/metrics/simple_metrics.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h b/vespalib/src/vespa/vespalib/metrics/simple_metrics.h index e558e93a638..e558e93a638 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics.h +++ b/vespalib/src/vespa/vespalib/metrics/simple_metrics.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp index 4b6b82697f7..4b6b82697f7 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp +++ b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h index fe3023ae020..fe3023ae020 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h +++ b/vespalib/src/vespa/vespalib/metrics/simple_metrics_manager.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp b/vespalib/src/vespa/vespalib/metrics/simple_tick.cpp index 0eed2ed3400..0eed2ed3400 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.cpp +++ b/vespalib/src/vespa/vespalib/metrics/simple_tick.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h b/vespalib/src/vespa/vespalib/metrics/simple_tick.h index 8ab18e74c83..8ab18e74c83 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/simple_tick.h +++ b/vespalib/src/vespa/vespalib/metrics/simple_tick.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp b/vespalib/src/vespa/vespalib/metrics/snapshots.cpp index 6ef9dca1e26..6ef9dca1e26 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.cpp +++ b/vespalib/src/vespa/vespalib/metrics/snapshots.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h b/vespalib/src/vespa/vespalib/metrics/snapshots.h index 2d2f29e7c23..2d2f29e7c23 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/snapshots.h +++ b/vespalib/src/vespa/vespalib/metrics/snapshots.h diff --git a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.cpp b/vespalib/src/vespa/vespalib/metrics/stable_store.cpp index dcb18d0cc82..dcb18d0cc82 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.cpp +++ b/vespalib/src/vespa/vespalib/metrics/stable_store.cpp diff --git a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h b/vespalib/src/vespa/vespalib/metrics/stable_store.h index 289d8f84d23..289d8f84d23 100644 --- a/staging_vespalib/src/vespa/vespalib/metrics/stable_store.h +++ b/vespalib/src/vespa/vespalib/metrics/stable_store.h diff --git a/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt b/vespalib/src/vespa/vespalib/net/http/CMakeLists.txt index f4d0ffdbc4a..b3e8d8c7a14 100644 --- a/staging_vespalib/src/vespa/vespalib/net/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/net/http/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(staging_vespalib_vespalib_net OBJECT +vespa_add_library(vespalib_vespalib_net_http OBJECT SOURCES component_config_producer.cpp generic_state_handler.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.cpp b/vespalib/src/vespa/vespalib/net/http/component_config_producer.cpp index 9011ff9abc9..9011ff9abc9 100644 --- a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.cpp +++ b/vespalib/src/vespa/vespalib/net/http/component_config_producer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h b/vespalib/src/vespa/vespalib/net/http/component_config_producer.h index 81ba8f8b03d..81ba8f8b03d 100644 --- a/staging_vespalib/src/vespa/vespalib/net/component_config_producer.h +++ b/vespalib/src/vespa/vespalib/net/http/component_config_producer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp b/vespalib/src/vespa/vespalib/net/http/generic_state_handler.cpp index 91fc6d7e617..91fc6d7e617 100644 --- a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.cpp +++ b/vespalib/src/vespa/vespalib/net/http/generic_state_handler.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h b/vespalib/src/vespa/vespalib/net/http/generic_state_handler.h index 6065aa165ec..6065aa165ec 100644 --- a/staging_vespalib/src/vespa/vespalib/net/generic_state_handler.h +++ b/vespalib/src/vespa/vespalib/net/http/generic_state_handler.h diff --git a/staging_vespalib/src/vespa/vespalib/net/health_producer.h b/vespalib/src/vespa/vespalib/net/http/health_producer.h index a16ffbd225f..a16ffbd225f 100644 --- a/staging_vespalib/src/vespa/vespalib/net/health_producer.h +++ b/vespalib/src/vespa/vespalib/net/http/health_producer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp b/vespalib/src/vespa/vespalib/net/http/http_server.cpp index f2c041cb648..f2c041cb648 100644 --- a/staging_vespalib/src/vespa/vespalib/net/http_server.cpp +++ b/vespalib/src/vespa/vespalib/net/http/http_server.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/http_server.h b/vespalib/src/vespa/vespalib/net/http/http_server.h index 3d4cd557b58..3d4cd557b58 100644 --- a/staging_vespalib/src/vespa/vespalib/net/http_server.h +++ b/vespalib/src/vespa/vespalib/net/http/http_server.h diff --git a/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h b/vespalib/src/vespa/vespalib/net/http/json_get_handler.h index d257bd0285f..d257bd0285f 100644 --- a/staging_vespalib/src/vespa/vespalib/net/json_get_handler.h +++ b/vespalib/src/vespa/vespalib/net/http/json_get_handler.h diff --git a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.cpp index 07b9306b5dc..07b9306b5dc 100644 --- a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.cpp +++ b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.h index adbe3010f43..adbe3010f43 100644 --- a/staging_vespalib/src/vespa/vespalib/net/json_handler_repo.h +++ b/vespalib/src/vespa/vespalib/net/http/json_handler_repo.h diff --git a/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h b/vespalib/src/vespa/vespalib/net/http/metrics_producer.h index d57797d75f6..f8b7b21022f 100644 --- a/staging_vespalib/src/vespa/vespalib/net/metrics_producer.h +++ b/vespalib/src/vespa/vespalib/net/http/metrics_producer.h @@ -9,7 +9,7 @@ namespace vespalib { struct MetricsProducer { virtual vespalib::string getMetrics(const vespalib::string &consumer) = 0; virtual vespalib::string getTotalMetrics(const vespalib::string &consumer) = 0; - virtual ~MetricsProducer() {} + virtual ~MetricsProducer() = default; }; } // namespace vespalib diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp b/vespalib/src/vespa/vespalib/net/http/simple_component_config_producer.cpp index 3fa5f09033f..3fa5f09033f 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.cpp +++ b/vespalib/src/vespa/vespalib/net/http/simple_component_config_producer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h b/vespalib/src/vespa/vespalib/net/http/simple_component_config_producer.h index 2f0cc0cf1fa..2f0cc0cf1fa 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_component_config_producer.h +++ b/vespalib/src/vespa/vespalib/net/http/simple_component_config_producer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp b/vespalib/src/vespa/vespalib/net/http/simple_health_producer.cpp index 3c1959608e9..3c1959608e9 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.cpp +++ b/vespalib/src/vespa/vespalib/net/http/simple_health_producer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h b/vespalib/src/vespa/vespalib/net/http/simple_health_producer.h index b2a7725b89e..b2a7725b89e 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_health_producer.h +++ b/vespalib/src/vespa/vespalib/net/http/simple_health_producer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp b/vespalib/src/vespa/vespalib/net/http/simple_metric_snapshot.cpp index cad2a3567aa..cad2a3567aa 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.cpp +++ b/vespalib/src/vespa/vespalib/net/http/simple_metric_snapshot.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h b/vespalib/src/vespa/vespalib/net/http/simple_metric_snapshot.h index e19f26bd5d0..e19f26bd5d0 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metric_snapshot.h +++ b/vespalib/src/vespa/vespalib/net/http/simple_metric_snapshot.h diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.cpp index 1c4539d70ee..1c4539d70ee 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.cpp +++ b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.h index ad463c63d57..ad463c63d57 100644 --- a/staging_vespalib/src/vespa/vespalib/net/simple_metrics_producer.h +++ b/vespalib/src/vespa/vespalib/net/http/simple_metrics_producer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp b/vespalib/src/vespa/vespalib/net/http/slime_explorer.cpp index dc61f673233..dc61f673233 100644 --- a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.cpp +++ b/vespalib/src/vespa/vespalib/net/http/slime_explorer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h b/vespalib/src/vespa/vespalib/net/http/slime_explorer.h index cbb0a89e36d..cbb0a89e36d 100644 --- a/staging_vespalib/src/vespa/vespalib/net/slime_explorer.h +++ b/vespalib/src/vespa/vespalib/net/http/slime_explorer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp b/vespalib/src/vespa/vespalib/net/http/state_api.cpp index ca00713352f..ca00713352f 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_api.cpp +++ b/vespalib/src/vespa/vespalib/net/http/state_api.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/state_api.h b/vespalib/src/vespa/vespalib/net/http/state_api.h index 2bdf320d091..2bdf320d091 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_api.h +++ b/vespalib/src/vespa/vespalib/net/http/state_api.h diff --git a/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp b/vespalib/src/vespa/vespalib/net/http/state_explorer.cpp index 627b70a92fc..627b70a92fc 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_explorer.cpp +++ b/vespalib/src/vespa/vespalib/net/http/state_explorer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/state_explorer.h b/vespalib/src/vespa/vespalib/net/http/state_explorer.h index 01cb94663b3..01cb94663b3 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_explorer.h +++ b/vespalib/src/vespa/vespalib/net/http/state_explorer.h diff --git a/staging_vespalib/src/vespa/vespalib/net/state_server.cpp b/vespalib/src/vespa/vespalib/net/http/state_server.cpp index 8d927387727..8d927387727 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_server.cpp +++ b/vespalib/src/vespa/vespalib/net/http/state_server.cpp diff --git a/staging_vespalib/src/vespa/vespalib/net/state_server.h b/vespalib/src/vespa/vespalib/net/http/state_server.h index 22152fe1a1f..22152fe1a1f 100644 --- a/staging_vespalib/src/vespa/vespalib/net/state_server.h +++ b/vespalib/src/vespa/vespalib/net/http/state_server.h diff --git a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt index a4b2192d98f..ddf01d5a935 100644 --- a/vespalib/src/vespa/vespalib/objects/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/objects/CMakeLists.txt @@ -1,7 +1,19 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. vespa_add_library(vespalib_vespalib_objects OBJECT SOURCES - nbostream.cpp + asciiserializer.cpp + deserializer.cpp + floatingpointtype.cpp hexdump.cpp + identifiable.cpp + nboserializer.cpp + nbostream.cpp + object2slime.cpp + objectdumper.cpp + objectoperation.cpp + objectpredicate.cpp + objectvisitor.cpp + serializer.cpp + visit.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.cpp b/vespalib/src/vespa/vespalib/objects/asciiserializer.cpp index fadaea9b054..fadaea9b054 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.cpp +++ b/vespalib/src/vespa/vespalib/objects/asciiserializer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.h b/vespalib/src/vespa/vespalib/objects/asciiserializer.h index 3964c52949d..3964c52949d 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/asciiserializer.h +++ b/vespalib/src/vespa/vespalib/objects/asciiserializer.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/deserializer.cpp b/vespalib/src/vespa/vespalib/objects/deserializer.cpp index f320b2d2489..f320b2d2489 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/deserializer.cpp +++ b/vespalib/src/vespa/vespalib/objects/deserializer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/deserializer.h b/vespalib/src/vespa/vespalib/objects/deserializer.h index e59d3e07581..e59d3e07581 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/deserializer.h +++ b/vespalib/src/vespa/vespalib/objects/deserializer.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/deserializer.hpp b/vespalib/src/vespa/vespalib/objects/deserializer.hpp index b90867fa153..b90867fa153 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/deserializer.hpp +++ b/vespalib/src/vespa/vespalib/objects/deserializer.hpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp b/vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp index b011dbdd23e..b011dbdd23e 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp +++ b/vespalib/src/vespa/vespalib/objects/floatingpointtype.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.h b/vespalib/src/vespa/vespalib/objects/floatingpointtype.h index d1fd80cdcd6..d1fd80cdcd6 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/floatingpointtype.h +++ b/vespalib/src/vespa/vespalib/objects/floatingpointtype.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp b/vespalib/src/vespa/vespalib/objects/identifiable.cpp index 224fda1a0a4..224fda1a0a4 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.cpp +++ b/vespalib/src/vespa/vespalib/objects/identifiable.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h b/vespalib/src/vespa/vespalib/objects/identifiable.h index 586fbc7ed0c..586fbc7ed0c 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.h +++ b/vespalib/src/vespa/vespalib/objects/identifiable.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/identifiable.hpp b/vespalib/src/vespa/vespalib/objects/identifiable.hpp index d86f5490bc9..d86f5490bc9 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/identifiable.hpp +++ b/vespalib/src/vespa/vespalib/objects/identifiable.hpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/ids.h b/vespalib/src/vespa/vespalib/objects/ids.h index 21d4d92aef1..21d4d92aef1 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/ids.h +++ b/vespalib/src/vespa/vespalib/objects/ids.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/nboserializer.cpp b/vespalib/src/vespa/vespalib/objects/nboserializer.cpp index 0508210bd09..0508210bd09 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/nboserializer.cpp +++ b/vespalib/src/vespa/vespalib/objects/nboserializer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/nboserializer.h b/vespalib/src/vespa/vespalib/objects/nboserializer.h index 4e11771d64b..4e11771d64b 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/nboserializer.h +++ b/vespalib/src/vespa/vespalib/objects/nboserializer.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/object2slime.cpp b/vespalib/src/vespa/vespalib/objects/object2slime.cpp index ee7b0501832..ee7b0501832 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/object2slime.cpp +++ b/vespalib/src/vespa/vespalib/objects/object2slime.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/object2slime.h b/vespalib/src/vespa/vespalib/objects/object2slime.h index ffab739ed15..ffab739ed15 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/object2slime.h +++ b/vespalib/src/vespa/vespalib/objects/object2slime.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectdumper.cpp b/vespalib/src/vespa/vespalib/objects/objectdumper.cpp index 61230ace2f9..61230ace2f9 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectdumper.cpp +++ b/vespalib/src/vespa/vespalib/objects/objectdumper.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectdumper.h b/vespalib/src/vespa/vespalib/objects/objectdumper.h index 0fb27c03ac2..0fb27c03ac2 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectdumper.h +++ b/vespalib/src/vespa/vespalib/objects/objectdumper.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectoperation.cpp b/vespalib/src/vespa/vespalib/objects/objectoperation.cpp index 0a84ad4b70f..0a84ad4b70f 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectoperation.cpp +++ b/vespalib/src/vespa/vespalib/objects/objectoperation.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectoperation.h b/vespalib/src/vespa/vespalib/objects/objectoperation.h index ba2820f1a85..ba2820f1a85 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectoperation.h +++ b/vespalib/src/vespa/vespalib/objects/objectoperation.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.cpp b/vespalib/src/vespa/vespalib/objects/objectpredicate.cpp index 61986ed4cf8..61986ed4cf8 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.cpp +++ b/vespalib/src/vespa/vespalib/objects/objectpredicate.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.h b/vespalib/src/vespa/vespalib/objects/objectpredicate.h index 12af9bab3e6..12af9bab3e6 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectpredicate.h +++ b/vespalib/src/vespa/vespalib/objects/objectpredicate.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.cpp b/vespalib/src/vespa/vespalib/objects/objectvisitor.cpp index 5466b19f6d7..5466b19f6d7 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.cpp +++ b/vespalib/src/vespa/vespalib/objects/objectvisitor.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.h b/vespalib/src/vespa/vespalib/objects/objectvisitor.h index dba731a8940..dba731a8940 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/objectvisitor.h +++ b/vespalib/src/vespa/vespalib/objects/objectvisitor.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/serializer.cpp b/vespalib/src/vespa/vespalib/objects/serializer.cpp index 3c1e22e52ac..3c1e22e52ac 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/serializer.cpp +++ b/vespalib/src/vespa/vespalib/objects/serializer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/serializer.h b/vespalib/src/vespa/vespalib/objects/serializer.h index f10d03b1c04..f10d03b1c04 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/serializer.h +++ b/vespalib/src/vespa/vespalib/objects/serializer.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/serializer.hpp b/vespalib/src/vespa/vespalib/objects/serializer.hpp index 87c02ddf693..87c02ddf693 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/serializer.hpp +++ b/vespalib/src/vespa/vespalib/objects/serializer.hpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.cpp b/vespalib/src/vespa/vespalib/objects/visit.cpp index 03f46d4c354..03f46d4c354 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/visit.cpp +++ b/vespalib/src/vespa/vespalib/objects/visit.cpp diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.h b/vespalib/src/vespa/vespalib/objects/visit.h index 87d42923d61..87d42923d61 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/visit.h +++ b/vespalib/src/vespa/vespalib/objects/visit.h diff --git a/staging_vespalib/src/vespa/vespalib/objects/visit.hpp b/vespalib/src/vespa/vespalib/objects/visit.hpp index e3a82b212c0..e3a82b212c0 100644 --- a/staging_vespalib/src/vespa/vespalib/objects/visit.hpp +++ b/vespalib/src/vespa/vespalib/objects/visit.hpp diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.h b/vespalib/src/vespa/vespalib/stllike/cache.h index bf58f1aef98..bf58f1aef98 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache.h +++ b/vespalib/src/vespa/vespalib/stllike/cache.h diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp b/vespalib/src/vespa/vespalib/stllike/cache.hpp index bb05d564f1f..bb05d564f1f 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache.hpp +++ b/vespalib/src/vespa/vespalib/stllike/cache.hpp diff --git a/staging_vespalib/src/vespa/vespalib/stllike/cache_stats.h b/vespalib/src/vespa/vespalib/stllike/cache_stats.h index 7c83da2b004..7c83da2b004 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/cache_stats.h +++ b/vespalib/src/vespa/vespalib/stllike/cache_stats.h diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h b/vespalib/src/vespa/vespalib/stllike/lrucache_map.h index ffb7a427d11..ffb7a427d11 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.h +++ b/vespalib/src/vespa/vespalib/stllike/lrucache_map.h diff --git a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp b/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp index 73e9956ffdf..73e9956ffdf 100644 --- a/staging_vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp +++ b/vespalib/src/vespa/vespalib/stllike/lrucache_map.hpp diff --git a/vespalib/src/vespa/vespalib/util/CMakeLists.txt b/vespalib/src/vespa/vespalib/util/CMakeLists.txt index f7bdd4427e3..59c05dffbc6 100644 --- a/vespalib/src/vespa/vespalib/util/CMakeLists.txt +++ b/vespalib/src/vespa/vespalib/util/CMakeLists.txt @@ -2,6 +2,7 @@ vespa_add_library(vespalib_vespalib_util OBJECT SOURCES active.cpp + adaptive_sequenced_executor.cpp address_space.cpp alloc.cpp approx.cpp @@ -12,32 +13,45 @@ vespa_add_library(vespalib_vespalib_util OBJECT benchmark_timer.cpp bfloat16.cpp binary_hamming_distance.cpp + bits.cpp blockingthreadstackexecutor.cpp box.cpp + cgroup_resource_limits.cpp classname.cpp + clock.cpp compress.cpp compressor.cpp count_down_latch.cpp cpu_usage.cpp + crc.cpp destructor_callbacks.cpp + doom.cpp + document_runnable.cpp dual_merge_director.cpp error.cpp exception.cpp exceptions.cpp executor_idle_tracking.cpp file_area_freelist.cpp + foregroundtaskexecutor.cpp gate.cpp gencnt.cpp generationhandler.cpp generationholder.cpp + growablebytebuffer.cpp hdr_abort.cpp host_name.cpp + jsonexception.cpp + jsonstream.cpp + jsonwriter.cpp invokeserviceimpl.cpp + isequencedtaskexecutor.cpp issue.cpp joinable.cpp latch.cpp left_right_heap.cpp lz4compressor.cpp + malloc_mmap_guard.cpp md5.c memoryusage.cpp mmap_file_allocator.cpp @@ -46,27 +60,36 @@ vespa_add_library(vespalib_vespalib_util OBJECT nice.cpp printable.cpp priority_queue.cpp + process_memory_stats.cpp + programoptions.cpp random.cpp rcuvector.cpp regexp.cpp require.cpp + resource_limits.cpp reusable_set.cpp reusable_set_handle.cpp reusable_set_pool.cpp round_up_to_page_size.cpp runnable.cpp runnable_pair.cpp + rusage.cpp sequence.cpp + sequencedtaskexecutor.cpp + sequencedtaskexecutorobserver.cpp sha1.cpp shared_operation_throttler.cpp shared_string_repo.cpp + shutdownguard.cpp sig_catch.cpp signalhandler.cpp simple_thread_bundle.cpp + singleexecutor.cpp small_vector.cpp stash.cpp string_hash.cpp stringfmt.cpp + testclock.cpp thread.cpp thread_bundle.cpp threadstackexecutor.cpp @@ -74,6 +97,8 @@ vespa_add_library(vespalib_vespalib_util OBJECT time.cpp unwind_message.cpp valgrind.cpp + xmlserializable.cpp + xmlstream.cpp zstdcompressor.cpp DEPENDS ) diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp index 6db97ff0761..6db97ff0761 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp +++ b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h index fbebf8b4e4c..fbebf8b4e4c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h +++ b/vespalib/src/vespa/vespalib/util/adaptive_sequenced_executor.h diff --git a/vespalib/src/vespa/vespalib/util/alloc.cpp b/vespalib/src/vespa/vespalib/util/alloc.cpp index 3f49fc4b427..d7e134cbb5a 100644 --- a/vespalib/src/vespa/vespalib/util/alloc.cpp +++ b/vespalib/src/vespa/vespalib/util/alloc.cpp @@ -234,8 +234,13 @@ createAutoAllocatorsWithDefault() { return tmp; } +AutoAllocatorsMapWithDefault & +availableAutoAllocators() { + static AutoAllocatorsMapWithDefault S_availableAutoAllocators = createAutoAllocatorsWithDefault(); + return S_availableAutoAllocators; +} + -AutoAllocatorsMapWithDefault _G_availableAutoAllocators = createAutoAllocatorsWithDefault(); alloc::HeapAllocator _G_heapAllocatorDefault; alloc::AlignedHeapAllocator _G_512BalignedHeapAllocator(512); alloc::AlignedHeapAllocator _G_1KalignedHeapAllocator(1_Ki); @@ -268,12 +273,12 @@ MMapAllocator::getDefault() { MemoryAllocator & AutoAllocator::getDefault() { - return *_G_availableAutoAllocators.second; + return *availableAutoAllocators().second; } MemoryAllocator & AutoAllocator::getAllocator(size_t mmapLimit, size_t alignment) { - return getAutoAllocator(_G_availableAutoAllocators.first, mmapLimit, alignment); + return getAutoAllocator(availableAutoAllocators().first, mmapLimit, alignment); } MemoryAllocator::PtrAndSize @@ -353,6 +358,9 @@ MMapAllocator::salloc(size_t sz, void * wantedAddress) store_relaxed(_G_hasHugePageFailureJustHappened, false); } #ifdef __linux__ + if (madvise(buf, sz, MADV_HUGEPAGE) != 0) { + // Just an advise, not everyone will listen... + } if (sz >= _G_MMapNoCoreLimit) { if (madvise(buf, sz, MADV_DONTDUMP) != 0) { LOG(warning, "Failed madvise(%p, %ld, MADV_DONTDUMP) = '%s'", buf, sz, FastOS_FileInterface::getLastErrorString().c_str()); @@ -408,10 +416,21 @@ void MMapAllocator::free(PtrAndSize alloc) const { void MMapAllocator::sfree(PtrAndSize alloc) { if (alloc.first != nullptr) { - int retval = madvise(alloc.first, alloc.second, MADV_DONTNEED); - assert(retval == 0); - retval = munmap(alloc.first, alloc.second); - assert(retval == 0); + int madvise_retval = madvise(alloc.first, alloc.second, MADV_DONTNEED); + if (madvise_retval != 0) { + std::error_code ec(errno, std::system_category()); + if (errno == EINVAL) { + LOG(debug, "madvise(%p, %lx)=%d, errno=%s", alloc.first, alloc.second, madvise_retval, ec.message().c_str()); + } else { + LOG(warning, "madvise(%p, %lx)=%d, errno=%s", alloc.first, alloc.second, madvise_retval, ec.message().c_str()); + } + } + int munmap_retval = munmap(alloc.first, alloc.second); + if (munmap_retval != 0) { + std::error_code ec(errno, std::system_category()); + LOG(warning, "munmap(%p, %lx)=%d, errno=%s", alloc.first, alloc.second, munmap_retval, ec.message().c_str()); + abort(); + } if (alloc.second >= _G_MMapLogLimit) { std::lock_guard guard(_G_lock); MMapInfo info = _G_HugeMappings[alloc.first]; diff --git a/staging_vespalib/src/vespa/vespalib/util/bits.cpp b/vespalib/src/vespa/vespalib/util/bits.cpp index 2d9848f3cdf..2d9848f3cdf 100644 --- a/staging_vespalib/src/vespa/vespalib/util/bits.cpp +++ b/vespalib/src/vespa/vespalib/util/bits.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/bits.h b/vespalib/src/vespa/vespalib/util/bits.h index fbe1a59f62a..fbe1a59f62a 100644 --- a/staging_vespalib/src/vespa/vespalib/util/bits.h +++ b/vespalib/src/vespa/vespalib/util/bits.h diff --git a/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp new file mode 100644 index 00000000000..35094c8aeaa --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.cpp @@ -0,0 +1,235 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "cgroup_resource_limits.h" +#include "round_up_to_page_size.h" +#include <vespa/vespalib/io/fileutil.h> +#include <algorithm> +#include <cmath> +#include <fstream> +#include <limits> +#include <sstream> + + +namespace vespalib { + +namespace { + +std::string +combine_paths(const std::string& base_path, const std::string& controller, const std::string &cgroup_path) +{ + std::ostringstream os; + os << base_path; + if (!controller.empty()) { + os << "/" << controller; + } + if (!cgroup_path.empty() && cgroup_path != "/") { + if (cgroup_path[0] != '/') { + os << "/"; + } + os << cgroup_path; + } + return os.str(); +} + +const std::string empty_std_string; + +} + +CGroupResourceLimits::CGroupResourceLimits() + : CGroupResourceLimits("/sys/fs/cgroup", "/proc/self/cgroup") +{ +} + +CGroupResourceLimits::CGroupResourceLimits(const std::string& base_path, const std::string& map_path) + : _memory_limit(), + _cpu_limit(), + _base_path(base_path), + _map_path(map_path), + _cgroup_v2_path(), + _cgroup_v1_paths() +{ + get_cgroup_paths(); + if (!_cgroup_v1_paths.empty()) { + get_limits_v1(); + } else { + get_limits_v2(); + } +} + +CGroupResourceLimits::~CGroupResourceLimits() = default; + +void +CGroupResourceLimits::apply_memory_limit(uint64_t memory_limit) +{ + if (!_memory_limit.has_value() || _memory_limit.value() > memory_limit) { + _memory_limit = memory_limit; + } +} + +void +CGroupResourceLimits::apply_cpu_limit(uint32_t cpu_limit) +{ + if (!_cpu_limit.has_value() || _cpu_limit.value() > cpu_limit) { + _cpu_limit = cpu_limit; + } +} + +void +CGroupResourceLimits::get_cgroup_paths() +{ + /* + * See manual page for cgroups(7) on linux for format of /proc/[pid]/cgroup. + */ + std::ifstream cg_self(_map_path); + std::string line; + while (std::getline(cg_self, line)) { + if (line.size() >= 3 && line.substr(0, 3) == "0::") { + _cgroup_v2_path = line.substr(3); + continue; + } + auto col_pos = line.find(':'); + if (col_pos != std::string::npos) { + auto name_pos = col_pos + 1; + col_pos = line.find(':', name_pos); + if (col_pos != std::string::npos) { + auto cg_path = line.substr(col_pos + 1); + auto comma_pos = line.find(',', name_pos); + while (comma_pos < col_pos) { + auto subsystem = line.substr(name_pos, comma_pos - name_pos); + _cgroup_v1_paths[subsystem] = cg_path; + name_pos = comma_pos + 1; + comma_pos = line.find(',', name_pos); + } + auto subsystem = line.substr(name_pos, col_pos - name_pos); + _cgroup_v1_paths[subsystem] = cg_path; + } + } + } +} + +void +CGroupResourceLimits::foreach_cgroup_level(const std::string& controller, const std::string& cgroup_path, const std::function<void(const std::string&)>& callback) +{ + auto dir = combine_paths(_base_path, controller, empty_std_string); + if (!isDirectory(dir)) { + return; + } + callback(dir); + if (cgroup_path.empty() || cgroup_path == "/") { + return; + } + auto slash_pos = cgroup_path.find('/', 1); + while (slash_pos != std::string::npos) { + dir = combine_paths(_base_path, controller, cgroup_path.substr(0, slash_pos)); + if (!isDirectory(dir)) { + return; + } + callback(dir); + slash_pos = cgroup_path.find('/', slash_pos + 1); + } + dir = combine_paths(_base_path, controller, cgroup_path); + if (isDirectory(dir)) { + callback(dir); + } +} + +void +CGroupResourceLimits::foreach_cgroup_v1_level(const std::string& controller, const std::function<void(const std::string&)>& callback) +{ + auto itr = _cgroup_v1_paths.find(controller); + if (itr != _cgroup_v1_paths.end()) { + foreach_cgroup_level(controller, itr->second, callback); + } else { + foreach_cgroup_level(controller, empty_std_string, callback); + } +} + +void +CGroupResourceLimits::foreach_cgroup_v2_level(const std::function<void(const std::string&)>& callback) +{ + foreach_cgroup_level(empty_std_string, _cgroup_v2_path, callback); +} + +void +CGroupResourceLimits::get_memory_limits_v1(const std::string& dir) +{ + std::ifstream limitfile(dir + "/memory.limit_in_bytes"); + uint64_t memory_limit = std::numeric_limits<uint64_t>::max(); + limitfile >> memory_limit; + if (limitfile.good() && (memory_limit < std::numeric_limits<int64_t>::max() - (round_up_to_page_size(1) - 1))) { + apply_memory_limit(memory_limit); + } +} + +void +CGroupResourceLimits::get_memory_limits_v1() +{ + foreach_cgroup_v1_level("memory", [this](const std::string& dir) { get_memory_limits_v1(dir); }); +} + +void +CGroupResourceLimits::get_cpu_limits_v1(const std::string& dir) +{ + int32_t cpu_cfs_period_us = 0; + int32_t cpu_cfs_quota_us = 0; + std::ifstream period_file(dir + "/cpu.cfs_period_us"); + std::ifstream quota_file(dir + "/cpu.cfs_quota_us"); + period_file >> cpu_cfs_period_us; + quota_file >> cpu_cfs_quota_us; + if (quota_file.good() && period_file.good() && cpu_cfs_quota_us >= 0 && cpu_cfs_period_us > 0) { + auto cpu = std::max(1.0, std::ceil(static_cast<double>(cpu_cfs_quota_us) / cpu_cfs_period_us)); + apply_cpu_limit(cpu); + } +} + +void +CGroupResourceLimits::get_cpu_limits_v1() +{ + foreach_cgroup_v1_level("cpu", [this](const std::string& dir) { get_cpu_limits_v1(dir); }); +} + +void +CGroupResourceLimits::get_limits_v1() +{ + get_memory_limits_v1(); + get_cpu_limits_v1(); +} + +void +CGroupResourceLimits::get_memory_limits_v2(const std::string& dir) +{ + std::ifstream limitfile(dir + "/memory.max"); + uint64_t memory_limit = std::numeric_limits<uint64_t>::max(); + limitfile >> memory_limit; + if (limitfile.good()) { + apply_memory_limit(memory_limit); + } +} + +void +CGroupResourceLimits::get_cpu_limits_v2(const std::string& dir) +{ + int32_t cpu_cfs_period_us = 0; + int32_t cpu_cfs_quota_us = 0; + std::ifstream cpu_max_file(dir + "/cpu.max"); + cpu_max_file >> cpu_cfs_quota_us >> cpu_cfs_period_us; + if (cpu_max_file.good() && cpu_cfs_quota_us >= 0 && cpu_cfs_period_us > 0) { + auto cpu = std::max(1.0, std::ceil(static_cast<double>(cpu_cfs_quota_us) / cpu_cfs_period_us)); + apply_cpu_limit(cpu); + } +} + +void +CGroupResourceLimits::get_limits_v2(const std::string& dir) +{ + get_memory_limits_v2(dir); + get_cpu_limits_v2(dir); +} + +void +CGroupResourceLimits::get_limits_v2() +{ + foreach_cgroup_v2_level([this](const std::string& dir) { get_limits_v2(dir); }); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h new file mode 100644 index 00000000000..72333195545 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/cgroup_resource_limits.h @@ -0,0 +1,68 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> +#include <functional> +#include <optional> +#include <string> +#include <map> + +namespace vespalib { + +/* + * Class for getting cgroup resource limits (memory and cpu resources). + * + * Cgroups version 1 and version 2 are handled, but a mix of cgroups version 1 + * and cgroups version 2 is not handled. + * + * Systemd resource control unit settings applied to cgroups (cf. + * systemd.resource-control(5) manual page on Linux) and + * container resource limit (set by e.g. docker and podman) applied to cgroups + * should be reflected in the sampled resource limits. + * + * The --cpus and --memory options for docker/podman create are reflected + * in cgroup limits that are sampled, but the cgroup limit affected by the + * --cpuset-cpus option is not yet sampled. + * + * For cgroups version 1, "memory.limit_in_bytes", "cpu.cfs_quota_us" and + * "cpu.cfs_period_us" are sampled. + * + * For cgroups version 2, "memory.max" and "cpu.max" are sampled. + * + * This is intended to be used by vespa-proton-bin running on the host + * or in a container, as a systemd service or not, with cgroups + * version 1 or cgroups version 2. + */ +class CGroupResourceLimits { + std::optional<uint64_t> _memory_limit; + std::optional<uint32_t> _cpu_limit; + std::string _base_path; // "/sys/fs/cgroup" + std::string _map_path; // "/proc/self/cgroup" + std::string _cgroup_v2_path; // Same cgroup path for all controllers + std::map<std::string, std::string> _cgroup_v1_paths; // controller -> cgroup path + + void apply_memory_limit(uint64_t memory_limit); + void apply_cpu_limit(uint32_t cpu_limit); + void get_cgroup_paths(); + void foreach_cgroup_level(const std::string& controller, const std::string& cgroup_path, const std::function<void(const std::string&)>& callback); + void foreach_cgroup_v1_level(const std::string& controller, const std::function<void(const std::string&)>& callback); + void foreach_cgroup_v2_level(const std::function<void(const std::string&)>& callback); + void get_memory_limits_v1(const std::string& dir); + void get_memory_limits_v1(); + void get_cpu_limits_v1(const std::string& dir); + void get_cpu_limits_v1(); + void get_limits_v1(); + void get_memory_limits_v2(const std::string& dir); + void get_cpu_limits_v2(const std::string& dir); + void get_limits_v2(const std::string& dir); + void get_limits_v2(); +public: + CGroupResourceLimits(); + CGroupResourceLimits(const std::string& base_path, const std::string& map_path); + ~CGroupResourceLimits(); + const std::optional<uint64_t>& get_memory_limit() const noexcept { return _memory_limit; } + const std::optional<uint32_t>& get_cpu_limit() const noexcept { return _cpu_limit; } +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.cpp b/vespalib/src/vespa/vespalib/util/clock.cpp index 4b6e1e8ab85..4b6e1e8ab85 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.cpp +++ b/vespalib/src/vespa/vespalib/util/clock.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/clock.h b/vespalib/src/vespa/vespalib/util/clock.h index 4cd579b8f69..4cd579b8f69 100644 --- a/staging_vespalib/src/vespa/vespalib/util/clock.h +++ b/vespalib/src/vespa/vespalib/util/clock.h diff --git a/staging_vespalib/src/vespa/vespalib/util/crc.cpp b/vespalib/src/vespa/vespalib/util/crc.cpp index 1a9baf8e600..1a9baf8e600 100644 --- a/staging_vespalib/src/vespa/vespalib/util/crc.cpp +++ b/vespalib/src/vespa/vespalib/util/crc.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/crc.h b/vespalib/src/vespa/vespalib/util/crc.h index f5acafeff36..f5acafeff36 100644 --- a/staging_vespalib/src/vespa/vespalib/util/crc.h +++ b/vespalib/src/vespa/vespalib/util/crc.h diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp b/vespalib/src/vespa/vespalib/util/document_runnable.cpp index d7534514f41..d7534514f41 100644 --- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.cpp +++ b/vespalib/src/vespa/vespalib/util/document_runnable.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/document_runnable.h b/vespalib/src/vespa/vespalib/util/document_runnable.h index 5ca344ea7ef..5ca344ea7ef 100644 --- a/staging_vespalib/src/vespa/vespalib/util/document_runnable.h +++ b/vespalib/src/vespa/vespalib/util/document_runnable.h diff --git a/staging_vespalib/src/vespa/vespalib/util/doom.cpp b/vespalib/src/vespa/vespalib/util/doom.cpp index 93ebe317ae0..93ebe317ae0 100644 --- a/staging_vespalib/src/vespa/vespalib/util/doom.cpp +++ b/vespalib/src/vespa/vespalib/util/doom.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/doom.h b/vespalib/src/vespa/vespalib/util/doom.h index e7db0795fb7..e7db0795fb7 100644 --- a/staging_vespalib/src/vespa/vespalib/util/doom.h +++ b/vespalib/src/vespa/vespalib/util/doom.h diff --git a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h b/vespalib/src/vespa/vespalib/util/foreground_thread_executor.h index c1e56572614..c1e56572614 100644 --- a/staging_vespalib/src/vespa/vespalib/util/foreground_thread_executor.h +++ b/vespalib/src/vespa/vespalib/util/foreground_thread_executor.h diff --git a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp b/vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp index ce5237f41c9..ce5237f41c9 100644 --- a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h b/vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h index 615bf62afe5..615bf62afe5 100644 --- a/staging_vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h +++ b/vespalib/src/vespa/vespalib/util/foregroundtaskexecutor.h diff --git a/vespalib/src/vespa/vespalib/util/generationholder.h b/vespalib/src/vespa/vespalib/util/generationholder.h index 0bc630404af..971304a2a11 100644 --- a/vespalib/src/vespa/vespalib/util/generationholder.h +++ b/vespalib/src/vespa/vespalib/util/generationholder.h @@ -29,16 +29,6 @@ public: size_t getSize() const { return _size; } }; -template<typename A> -class GenerationHeldAlloc : public GenerationHeldBase -{ -public: - GenerationHeldAlloc(A & alloc) : GenerationHeldBase(alloc.size()), _alloc() { _alloc.swap(alloc); } - virtual ~GenerationHeldAlloc() { } -private: - A _alloc; -}; - /* * GenerationHolder is meant to hold large elements until readers can * no longer access them. diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp b/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp index 424ad7ba470..424ad7ba470 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp +++ b/vespalib/src/vespa/vespalib/util/growablebytebuffer.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h b/vespalib/src/vespa/vespalib/util/growablebytebuffer.h index b0fb30606d4..b0fb30606d4 100644 --- a/staging_vespalib/src/vespa/vespalib/util/growablebytebuffer.h +++ b/vespalib/src/vespa/vespalib/util/growablebytebuffer.h diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp b/vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp index b31d72da3b1..b31d72da3b1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h b/vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h index ff90556e3e4..ff90556e3e4 100644 --- a/staging_vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h +++ b/vespalib/src/vespa/vespalib/util/isequencedtaskexecutor.h diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonexception.cpp b/vespalib/src/vespa/vespalib/util/jsonexception.cpp index 5c04aefe2d1..5c04aefe2d1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonexception.cpp +++ b/vespalib/src/vespa/vespalib/util/jsonexception.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonexception.h b/vespalib/src/vespa/vespalib/util/jsonexception.h index 3bf954420a6..3bf954420a6 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonexception.h +++ b/vespalib/src/vespa/vespalib/util/jsonexception.h diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp b/vespalib/src/vespa/vespalib/util/jsonstream.cpp index 135611e975f..135611e975f 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonstream.cpp +++ b/vespalib/src/vespa/vespalib/util/jsonstream.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonstream.h b/vespalib/src/vespa/vespalib/util/jsonstream.h index d60151f0478..d60151f0478 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonstream.h +++ b/vespalib/src/vespa/vespalib/util/jsonstream.h diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp b/vespalib/src/vespa/vespalib/util/jsonwriter.cpp index eb56d8b367a..eb56d8b367a 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.cpp +++ b/vespalib/src/vespa/vespalib/util/jsonwriter.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h b/vespalib/src/vespa/vespalib/util/jsonwriter.h index 57380b46e5c..57380b46e5c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/jsonwriter.h +++ b/vespalib/src/vespa/vespalib/util/jsonwriter.h diff --git a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp index 67181dfd16f..67181dfd16f 100644 --- a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp +++ b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h index 03e6d38c03c..03e6d38c03c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h +++ b/vespalib/src/vespa/vespalib/util/malloc_mmap_guard.h diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h b/vespalib/src/vespa/vespalib/util/polymorphicarray.h index de93ae03657..de93ae03657 100644 --- a/staging_vespalib/src/vespa/vespalib/util/polymorphicarray.h +++ b/vespalib/src/vespa/vespalib/util/polymorphicarray.h diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h b/vespalib/src/vespa/vespalib/util/polymorphicarraybase.h index f3f63f6a43c..f3f63f6a43c 100644 --- a/staging_vespalib/src/vespa/vespalib/util/polymorphicarraybase.h +++ b/vespalib/src/vespa/vespalib/util/polymorphicarraybase.h diff --git a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h b/vespalib/src/vespa/vespalib/util/polymorphicarrays.h index 55f64ec51cd..55f64ec51cd 100644 --- a/staging_vespalib/src/vespa/vespalib/util/polymorphicarrays.h +++ b/vespalib/src/vespa/vespalib/util/polymorphicarrays.h diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index f7e8e087727..f7e8e087727 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/vespalib/src/vespa/vespalib/util/process_memory_stats.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h b/vespalib/src/vespa/vespalib/util/process_memory_stats.h index 000c0942905..000c0942905 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h +++ b/vespalib/src/vespa/vespalib/util/process_memory_stats.h diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp b/vespalib/src/vespa/vespalib/util/programoptions.cpp index 9ea7c1648d1..9ea7c1648d1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/programoptions.cpp +++ b/vespalib/src/vespa/vespalib/util/programoptions.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/programoptions.h b/vespalib/src/vespa/vespalib/util/programoptions.h index 0c75f9b2c72..0c75f9b2c72 100644 --- a/staging_vespalib/src/vespa/vespalib/util/programoptions.h +++ b/vespalib/src/vespa/vespalib/util/programoptions.h diff --git a/vespalib/src/vespa/vespalib/util/resource_limits.cpp b/vespalib/src/vespa/vespalib/util/resource_limits.cpp new file mode 100644 index 00000000000..85c479a67b6 --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/resource_limits.cpp @@ -0,0 +1,33 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "resource_limits.h" +#include "cgroup_resource_limits.h" +#include <unistd.h> +#include <thread> + +namespace vespalib { + +ResourceLimits::ResourceLimits(uint64_t memory, uint32_t cpu) + : _memory(memory), + _cpu(cpu) +{ +} + +ResourceLimits +ResourceLimits::create() +{ + uint64_t memory = sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE); + uint32_t cpu = std::thread::hardware_concurrency(); + CGroupResourceLimits cgroup_limits; + auto& cg_memory = cgroup_limits.get_memory_limit(); + auto& cg_cpu = cgroup_limits.get_cpu_limit(); + if (cg_memory.has_value() && cg_memory.value() < memory) { + memory = cg_memory.value(); + } + if (cg_cpu.has_value() && cg_cpu.value() < cpu) { + cpu = cg_cpu.value(); + } + return ResourceLimits(memory, cpu); +} + +} diff --git a/vespalib/src/vespa/vespalib/util/resource_limits.h b/vespalib/src/vespa/vespalib/util/resource_limits.h new file mode 100644 index 00000000000..8e1c7a7c34f --- /dev/null +++ b/vespalib/src/vespa/vespalib/util/resource_limits.h @@ -0,0 +1,27 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include <cstdint> + +namespace vespalib { + +/* + * Class for getting resource limits. Memory limit is first sampled by + * using sysconf to get page size and number of physical pages. + * Cpu limit is first sampled by calling std::thread::hardware_concurrency(). + * Both limits can be adjusted downwards by applying some of the cgroup limits + * for the current process, cf. CGroupResourceLimits). + */ +class ResourceLimits { + uint64_t _memory; + uint32_t _cpu; + + ResourceLimits(uint64_t memory, uint32_t cpu); +public: + static ResourceLimits create(); + uint64_t memory() const noexcept { return _memory; } + uint32_t cpu() const noexcept { return _cpu; } +}; + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp b/vespalib/src/vespa/vespalib/util/rusage.cpp index ed0ec125dcf..ed0ec125dcf 100644 --- a/staging_vespalib/src/vespa/vespalib/util/rusage.cpp +++ b/vespalib/src/vespa/vespalib/util/rusage.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/rusage.h b/vespalib/src/vespa/vespalib/util/rusage.h index 4c741f7699d..4c741f7699d 100644 --- a/staging_vespalib/src/vespa/vespalib/util/rusage.h +++ b/vespalib/src/vespa/vespalib/util/rusage.h diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp index e12d2065d9f..e12d2065d9f 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h index a4b1b82aacf..a4b1b82aacf 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h +++ b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutor.h diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp index d81b8ec1db6..d81b8ec1db6 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp +++ b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h index 1d54283c393..1d54283c393 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h +++ b/vespalib/src/vespa/vespalib/util/sequencedtaskexecutorobserver.h diff --git a/staging_vespalib/src/vespa/vespalib/util/shutdownguard.cpp b/vespalib/src/vespa/vespalib/util/shutdownguard.cpp index 12e58898c06..12e58898c06 100644 --- a/staging_vespalib/src/vespa/vespalib/util/shutdownguard.cpp +++ b/vespalib/src/vespa/vespalib/util/shutdownguard.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/shutdownguard.h b/vespalib/src/vespa/vespalib/util/shutdownguard.h index d76d4deb5d2..d76d4deb5d2 100644 --- a/staging_vespalib/src/vespa/vespalib/util/shutdownguard.h +++ b/vespalib/src/vespa/vespalib/util/shutdownguard.h diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp b/vespalib/src/vespa/vespalib/util/singleexecutor.cpp index 21ed90c3d22..21ed90c3d22 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.cpp +++ b/vespalib/src/vespa/vespalib/util/singleexecutor.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h b/vespalib/src/vespa/vespalib/util/singleexecutor.h index dd755a76302..dd755a76302 100644 --- a/staging_vespalib/src/vespa/vespalib/util/singleexecutor.h +++ b/vespalib/src/vespa/vespalib/util/singleexecutor.h diff --git a/staging_vespalib/src/vespa/vespalib/util/sort.h b/vespalib/src/vespa/vespalib/util/sort.h index ce3f6772ef1..ce3f6772ef1 100644 --- a/staging_vespalib/src/vespa/vespalib/util/sort.h +++ b/vespalib/src/vespa/vespalib/util/sort.h diff --git a/staging_vespalib/src/vespa/vespalib/util/testclock.cpp b/vespalib/src/vespa/vespalib/util/testclock.cpp index bc5d37ca437..bc5d37ca437 100644 --- a/staging_vespalib/src/vespa/vespalib/util/testclock.cpp +++ b/vespalib/src/vespa/vespalib/util/testclock.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/testclock.h b/vespalib/src/vespa/vespalib/util/testclock.h index 9446ff32cb2..9446ff32cb2 100644 --- a/staging_vespalib/src/vespa/vespalib/util/testclock.h +++ b/vespalib/src/vespa/vespalib/util/testclock.h diff --git a/staging_vespalib/src/vespa/vespalib/util/varholder.h b/vespalib/src/vespa/vespalib/util/varholder.h index d92c00e0081..d92c00e0081 100644 --- a/staging_vespalib/src/vespa/vespalib/util/varholder.h +++ b/vespalib/src/vespa/vespalib/util/varholder.h diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp b/vespalib/src/vespa/vespalib/util/xmlserializable.cpp index c687609df74..c687609df74 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.cpp +++ b/vespalib/src/vespa/vespalib/util/xmlserializable.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h b/vespalib/src/vespa/vespalib/util/xmlserializable.h index 76f0ad1fa3a..76f0ad1fa3a 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlserializable.h +++ b/vespalib/src/vespa/vespalib/util/xmlserializable.h diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp b/vespalib/src/vespa/vespalib/util/xmlstream.cpp index bdc09da127b..bdc09da127b 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlstream.cpp +++ b/vespalib/src/vespa/vespalib/util/xmlstream.cpp diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.h b/vespalib/src/vespa/vespalib/util/xmlstream.h index b197ff5c7c8..b197ff5c7c8 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlstream.h +++ b/vespalib/src/vespa/vespalib/util/xmlstream.h diff --git a/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp b/vespalib/src/vespa/vespalib/util/xmlstream.hpp index 82f1146df29..82f1146df29 100644 --- a/staging_vespalib/src/vespa/vespalib/util/xmlstream.hpp +++ b/vespalib/src/vespa/vespalib/util/xmlstream.hpp diff --git a/vespamalloc/src/vespamalloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/CMakeLists.txt index 8ee0f150fb1..2bbf9e59fec 100644 --- a/vespamalloc/src/vespamalloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/CMakeLists.txt @@ -42,3 +42,10 @@ vespa_add_library(vespammap DEPENDS dl ) +vespa_add_library(vespa_load_as_huge + SOURCES + $<TARGET_OBJECTS:vespamalloc_load_as_huge> + INSTALL lib64/vespa/malloc + DEPENDS + dl +) diff --git a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt index 985cd9948ad..5f50cd97aee 100644 --- a/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt +++ b/vespamalloc/src/vespamalloc/malloc/CMakeLists.txt @@ -69,3 +69,8 @@ vespa_add_library(vespamalloc_mmap OBJECT mmap.cpp DEPENDS ) +vespa_add_library(vespamalloc_load_as_huge OBJECT + SOURCES + load_as_huge.cpp + DEPENDS +) diff --git a/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp b/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp new file mode 100644 index 00000000000..00916ec9fac --- /dev/null +++ b/vespamalloc/src/vespamalloc/malloc/load_as_huge.cpp @@ -0,0 +1,133 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include <cstdio> +#include <cerrno> +#include <cassert> +#include <cstring> +#include <link.h> +#include <sys/mman.h> + +/** + * This is experimental code that will map code segments in binary and dso into + * anonymous mappings prefering huge page mappings. + */ +namespace { + +constexpr size_t HUGEPAGE_SIZE = 0x200000; + +void * +mmap_huge(size_t sz) { + assert ((sz % HUGEPAGE_SIZE) == 0); + void * mem = mmap(nullptr, sz, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0); + assert(mem != MAP_FAILED); + if (madvise(mem, sz, MADV_HUGEPAGE) != 0) { + perror("load_as_huge.cpp:mmap_huge => madvise( MADV_HUGEPAGE) failed"); + } + return mem; +} + +size_t round_huge_down(size_t v) { return v & ~(HUGEPAGE_SIZE - 1); } +size_t round_huge_up(size_t v) { return round_huge_down(v + (HUGEPAGE_SIZE - 1)); } + +void +non_optimized_non_inlined_memcpy(void *dest_in, const void *src_in, size_t n) __attribute__((noinline, optimize(1))) ; + +// Simple memcpy replacement to avoid calling code in other dso. +void +non_optimized_non_inlined_memcpy(void *dest_in, const void *src_in, size_t n) { + char *dest = static_cast<char *>(dest_in); + const char *src = static_cast<const char *>(src_in); + for (size_t i(0); i < n ; i++) { + dest[i] = src[i]; + } +} + +/** + * Make a large mapping if code is larger than HUGEPAGE_SIZE and copies the content of the various segments. + * Then remaps the areas back to its original location. + */ +bool +remap_segments(size_t base_vaddr, const Elf64_Phdr * segments, size_t count) { + assert(count > 0); + const Elf64_Phdr & first = segments[0]; + const Elf64_Phdr & last = segments[count - 1]; + size_t start_vaddr = base_vaddr + first.p_vaddr; + size_t end_vaddr = base_vaddr + last.p_vaddr + last.p_memsz; + if (end_vaddr - start_vaddr < HUGEPAGE_SIZE) { + return false; + } + + size_t huge_start = round_huge_down(start_vaddr); + size_t huge_end = round_huge_up(end_vaddr); + size_t huge_size = huge_end - huge_start; + char * new_huge = static_cast<char *>(mmap_huge(huge_size)); + char * new_huge_end = new_huge + huge_size; + char * last_end = new_huge; + for (size_t i(0); i < count; i++) { + size_t vaddr = base_vaddr + segments[i].p_vaddr; + size_t huge_offset = vaddr - huge_start; + char * dest = new_huge + huge_offset; + assert(dest >= last_end); + if (dest > last_end) { + int munmap_retval = munmap(last_end, dest - last_end); + assert(munmap_retval == 0); + } + size_t sz = segments[i].p_memsz; + last_end = dest + sz; + + if (madvise(dest, sz, MADV_HUGEPAGE) != 0) { + perror("load_as_huge.cpp:mmap_huge => madvise( MADV_HUGEPAGE) failed"); + } + non_optimized_non_inlined_memcpy(dest, reinterpret_cast<void*>(vaddr), sz); + int prot = PROT_READ; + if (segments[i].p_flags & PF_X) prot|= PROT_EXEC; + if (segments[i].p_flags & PF_W) prot|= PROT_WRITE; + int mprotect_retval = mprotect(dest, sz, prot); + if (mprotect_retval != 0) { + fprintf(stderr, "mprotect(%p, %ld, %x) FAILED = %d, errno= %d = %s\n", dest, sz, prot, mprotect_retval, errno, strerror(errno)); + } + void * remapped = mremap(dest, sz, sz, MREMAP_FIXED | MREMAP_MAYMOVE, vaddr); + assert(remapped != MAP_FAILED); + assert(remapped == reinterpret_cast<void *>(vaddr)); + fprintf(stderr, "remapped dest=%p, size=%lu to %p\n", dest, sz, remapped); + } + assert(new_huge_end >= last_end); + if (new_huge_end > last_end) { + int munmap_retval = munmap(last_end, new_huge_end - last_end); + assert(munmap_retval); + } + return true; +} + +int +remapElfHeader(struct dl_phdr_info *info, size_t info_size, void *data) { + (void) info_size; + (void) data; + fprintf(stderr, "processing elf header '%s' with %d entries, start=%lx\n", + info->dlpi_name, info->dlpi_phnum, info->dlpi_addr); + for (int i = 0; i < info->dlpi_phnum; i++) { + const Elf64_Phdr &phdr = info->dlpi_phdr[i]; + //fprintf(stderr, "p_vaddr=%lx p_paddr=%lx, p_offset=%lx p_filesz=%lx, p_memsz=%lx, allign=%lu type=%d flags=%x\n", + // phdr.p_vaddr, phdr.p_paddr, phdr.p_offset, phdr.p_filesz, phdr.p_memsz, phdr.p_align, phdr.p_type, phdr.p_flags); + if ((phdr.p_type == PT_LOAD) && (phdr.p_flags == (PF_R | PF_X))) { + //void *vaddr = reinterpret_cast<void *>(info->dlpi_addr + phdr.p_vaddr); + //uint64_t size = phdr.p_filesz; + //fprintf(stderr, "LOAD_RX: vaddr=%lx p_filesz=%lu, p_memsz=%lu\n", phdr.p_vaddr, phdr.p_filesz, phdr.p_memsz); + remap_segments(info->dlpi_addr, &phdr, 1); + } + } + return 0; +} + +} + +extern "C" int remapTextWithHugePages(); + +int +remapTextWithHugePages() { + int retval = dl_iterate_phdr(remapElfHeader, nullptr); + fprintf(stderr, "dl_iterate_phdr() = %d\n", retval); + return retval; +} + +static long num_huge_code_pages = remapTextWithHugePages(); diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp index eadd5aee337..1fb6aa74af9 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -64,6 +64,9 @@ MMapPool::mmap(size_t sz) { } } #ifdef __linux__ + if (madvise(buf, sz, MADV_HUGEPAGE) != 0) { + // Just an advise, not everyone will listen... + } if (sz >= _G_bigBlockLimit) { if (madvise(buf, sz, MADV_DONTDUMP) != 0) { std::error_code ec(errno, std::system_category()); diff --git a/vespamalloc/src/vespamalloc/util/osmem.cpp b/vespamalloc/src/vespamalloc/util/osmem.cpp index e0cfbc36938..2cb0eb78ad2 100644 --- a/vespamalloc/src/vespamalloc/util/osmem.cpp +++ b/vespamalloc/src/vespamalloc/util/osmem.cpp @@ -214,6 +214,9 @@ MmapMemory::getBasePages(size_t len, int mmapOpt, int fd, size_t offset) ok = (mem == wanted); } if (mem != MAP_FAILED) { + if (madvise(mem, len, MADV_HUGEPAGE) != 0) { + // Just an advise, not everyone will listen... + } if (getStart() == nullptr) { setStart(mem); // assumes len parameter is always multiple of the current block size. diff --git a/vsm/.gitignore b/vsm/.gitignore deleted file mode 100644 index 3969348a427..00000000000 --- a/vsm/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -bin -lib -/target -/pom.xml.build -Makefile diff --git a/vsm/CMakeLists.txt b/vsm/CMakeLists.txt deleted file mode 100644 index e1607ff27aa..00000000000 --- a/vsm/CMakeLists.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_define_module( - DEPENDS - fastos - vespalog - fastlib_fast - config_cloudconfig - vespalib - document - searchlib - searchsummary - - TESTS - src/tests/charbuffer - src/tests/docsum - src/tests/document - src/tests/searcher - src/tests/textutil - - LIBS - src/vespa/vsm - src/vespa/vsm/common - src/vespa/vsm/config - src/vespa/vsm/searcher - src/vespa/vsm/vsm -) diff --git a/vsm/OWNERS b/vsm/OWNERS deleted file mode 100644 index 465c0e30113..00000000000 --- a/vsm/OWNERS +++ /dev/null @@ -1,2 +0,0 @@ -baldersheim -geirst diff --git a/vsm/README b/vsm/README deleted file mode 100644 index a7686213183..00000000000 --- a/vsm/README +++ /dev/null @@ -1 +0,0 @@ -Vespa Streaming Matcher applications diff --git a/vsm/src/.gitignore b/vsm/src/.gitignore deleted file mode 100644 index d185ea18fd0..00000000000 --- a/vsm/src/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -*.dsp -Makefile.ini -config_command.sh -project.dsw diff --git a/vsm/src/tests/utilapps/.gitignore b/vsm/src/tests/utilapps/.gitignore deleted file mode 100644 index e69de29bb2d..00000000000 --- a/vsm/src/tests/utilapps/.gitignore +++ /dev/null diff --git a/vsm/src/vespa/vsm/CMakeLists.txt b/vsm/src/vespa/vsm/CMakeLists.txt deleted file mode 100644 index 8ceabbe494f..00000000000 --- a/vsm/src/vespa/vsm/CMakeLists.txt +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(vsm - SOURCES - $<TARGET_OBJECTS:vsm_vconfig> - $<TARGET_OBJECTS:vsm_vsmbase> - $<TARGET_OBJECTS:vsm_vsmcommon> - $<TARGET_OBJECTS:vsm_vsmsearcher> - INSTALL lib64 - DEPENDS -) diff --git a/vsm/testrun/.gitignore b/vsm/testrun/.gitignore deleted file mode 100644 index 3b525d55e24..00000000000 --- a/vsm/testrun/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -test-report.html -test-report.html.* -test.*.*.file.* -test.*.*.files.html -tmp.* -/test.*.*.result -Makefile diff --git a/zkfacade/CMakeLists.txt b/zkfacade/CMakeLists.txt index 4ea012aa9a5..a29a66cccfc 100644 --- a/zkfacade/CMakeLists.txt +++ b/zkfacade/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zkfacade) +install_jar(zkfacade-jar-with-dependencies.jar) diff --git a/zookeeper-command-line-client/CMakeLists.txt b/zookeeper-command-line-client/CMakeLists.txt index 81a100d1b25..8140822388c 100644 --- a/zookeeper-command-line-client/CMakeLists.txt +++ b/zookeeper-command-line-client/CMakeLists.txt @@ -1,5 +1,5 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-command-line-client) +install_jar(zookeeper-command-line-client-jar-with-dependencies.jar) vespa_install_script(src/main/sh/vespa-zkcat vespa-zkcat bin) vespa_install_script(src/main/sh/vespa-zkcli vespa-zkcli bin) diff --git a/zookeeper-server/CMakeLists.txt b/zookeeper-server/CMakeLists.txt index a4e4ddaeb2d..42ecff1cc96 100644 --- a/zookeeper-server/CMakeLists.txt +++ b/zookeeper-server/CMakeLists.txt @@ -1,3 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. add_subdirectory(zookeeper-server-common) -add_subdirectory(zookeeper-server-3.7.0) +add_subdirectory(zookeeper-server) +add_subdirectory(zookeeper-server-3.7.1) diff --git a/zookeeper-server/README b/zookeeper-server/README index 9bf49ab7f87..2f4845dad29 100644 --- a/zookeeper-server/README +++ b/zookeeper-server/README @@ -1 +1,8 @@ Vespa ZooKeeper server interface + +Subdirectory zookeeper-server has code for the zookeeper server +version we use by default, testing out new versions can be done by +copying zookeeper-server dir to zookeeper-server-<version> and +updating CMakeLists.txt and pom.xml in that directory and pom.xml and +CMakeLists.txt in this directory. Selecting zookeeper server version +is done with feature flag ZOOKEEPER_SERVER_VERSION. diff --git a/zookeeper-server/pom.xml b/zookeeper-server/pom.xml index 3efdde2cd81..34f4c0e9191 100644 --- a/zookeeper-server/pom.xml +++ b/zookeeper-server/pom.xml @@ -8,12 +8,13 @@ <version>7-SNAPSHOT</version> <relativePath>../parent/pom.xml</relativePath> </parent> - <artifactId>zookeeper-server</artifactId> + <artifactId>zookeeper-server-parent</artifactId> <packaging>pom</packaging> <version>7-SNAPSHOT</version> <modules> <module>zookeeper-server-common</module> - <module>zookeeper-server-3.7.0</module> + <module>zookeeper-server</module> + <module>zookeeper-server-3.7.1</module> </modules> <dependencies> <dependency> diff --git a/fastlib/src/vespa/fastlib/util/CMakeLists.txt b/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt index 56c7baabee2..6afd812ef07 100644 --- a/fastlib/src/vespa/fastlib/util/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server-3.7.1/CMakeLists.txt @@ -1,6 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(fastlib_util - SOURCES - INSTALL lib64 - DEPENDS -) +install_jar(zookeeper-server-3.7.1-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.7.1/pom.xml b/zookeeper-server/zookeeper-server-3.7.1/pom.xml new file mode 100644 index 00000000000..ed677ef8601 --- /dev/null +++ b/zookeeper-server/zookeeper-server-3.7.1/pom.xml @@ -0,0 +1,123 @@ +<?xml version="1.0"?> +<!-- Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. --> +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-parent</artifactId> + <version>7-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + <artifactId>zookeeper-server-3.7.1</artifactId> + <packaging>container-plugin</packaging> + <version>7-SNAPSHOT</version> + <properties> + <zookeeper.version>3.7.1</zookeeper.version> + </properties> + <dependencies> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-server-common</artifactId> + <version>${project.version}</version> + </dependency> + <dependency> + <groupId>com.yahoo.vespa</groupId> + <artifactId>zookeeper-client-common</artifactId> + <version>${project.version}</version> + <exclusions> + <exclusion> + <!-- Don't use ZK version from zookeeper-client-common --> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper</artifactId> + <version>${zookeeper.version}</version> + <exclusions> + <!-- + Container provides wiring for all common log libraries + Duplicate embedding results in various warnings being printed to stderr + --> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-log4j12</artifactId> + </exclusion> + <exclusion> + <groupId>log4j</groupId> + <artifactId>log4j</artifactId> + </exclusion> + </exclusions> + </dependency> + <!-- snappy-java and metrics-core are included here + to be able to work with ZooKeeper 3.7.0 due to + class loading issues --> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-compiler-plugin</artifactId> + <configuration> + <compilerArgs> + <arg>-Xlint:all</arg> + </compilerArgs> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <redirectTestOutputToFile>${test.hide}</redirectTestOutputToFile> + <forkMode>once</forkMode> + <systemPropertyVariables> + <zk-version>${zookeeper.version}</zk-version> + </systemPropertyVariables> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-install-plugin</artifactId> + <configuration> + <updateReleaseInfo>true</updateReleaseInfo> + </configuration> + </plugin> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>bundle-plugin</artifactId> + <extensions>true</extensions> + <configuration> + <importPackage>com.sun.management</importPackage> + <bundleSymbolicName>zookeeper-server</bundleSymbolicName> + </configuration> + </plugin> + </plugins> + </build> +</project> diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java index e94110af2fb..e94110af2fb 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java index 66742b0e05b..66742b0e05b 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java index 47ec03367c1..47ec03367c1 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java index ae7bf8d84f5..ae7bf8d84f5 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java index 48f95d28910..48f95d28910 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java index 33ec9b1303a..33ec9b1303a 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/org/apache/zookeeper/common/NetUtils.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/common/NetUtils.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java index fdfe0fe8467..fdfe0fe8467 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java diff --git a/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java index db643d76e0d..db643d76e0d 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java +++ b/zookeeper-server/zookeeper-server-3.7.1/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java diff --git a/zookeeper-server/zookeeper-server-common/CMakeLists.txt b/zookeeper-server/zookeeper-server-common/CMakeLists.txt index 6f8ea04c263..9f3d26e836f 100644 --- a/zookeeper-server/zookeeper-server-common/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server-common/CMakeLists.txt @@ -1,2 +1,2 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-common) +install_jar(zookeeper-server-common-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-common/pom.xml b/zookeeper-server/zookeeper-server-common/pom.xml index a9f74cb1d0e..82494e7bd14 100644 --- a/zookeeper-server/zookeeper-server-common/pom.xml +++ b/zookeeper-server/zookeeper-server-common/pom.xml @@ -4,7 +4,7 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server</artifactId> + <artifactId>zookeeper-server-parent</artifactId> <version>7-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> diff --git a/zookeeper-server/zookeeper-server-3.7.0/CMakeLists.txt b/zookeeper-server/zookeeper-server/CMakeLists.txt index 8a41fc8b8fa..98cb495e4b6 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/CMakeLists.txt +++ b/zookeeper-server/zookeeper-server/CMakeLists.txt @@ -1,4 +1,4 @@ # Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -install_fat_java_artifact(zookeeper-server-3.7.0) -# Needs to be included when this is the wanted default version (and symlinks for other versions need to be removed) +install_jar(zookeeper-server-3.7.0-jar-with-dependencies.jar) +# Make symlink so that we have a default version, should be done only in zookeeper-server module install_symlink(lib/jars/zookeeper-server-3.7.0-jar-with-dependencies.jar lib/jars/zookeeper-server-jar-with-dependencies.jar) diff --git a/zookeeper-server/zookeeper-server-3.7.0/pom.xml b/zookeeper-server/zookeeper-server/pom.xml index 8daa6003f1e..01cd666d5ab 100644 --- a/zookeeper-server/zookeeper-server-3.7.0/pom.xml +++ b/zookeeper-server/zookeeper-server/pom.xml @@ -4,7 +4,7 @@ <modelVersion>4.0.0</modelVersion> <parent> <groupId>com.yahoo.vespa</groupId> - <artifactId>zookeeper-server</artifactId> + <artifactId>zookeeper-server-parent</artifactId> <version>7-SNAPSHOT</version> <relativePath>../pom.xml</relativePath> </parent> diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java new file mode 100644 index 00000000000..e94110af2fb --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/ReconfigurableVespaZooKeeperServer.java @@ -0,0 +1,43 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; + +import java.nio.file.Path; +import java.time.Duration; +import java.util.concurrent.atomic.AtomicReference; + +/** + * Starts or reconfigures zookeeper cluster. + * The QuorumPeer conditionally created here is owned by the Reconfigurer; + * when it already has a peer, that peer is used here in case start or shutdown is required. + * + * @author hmusum + */ +public class ReconfigurableVespaZooKeeperServer extends AbstractComponent implements VespaZooKeeperServer { + + private QuorumPeer peer; + + @Inject + public ReconfigurableVespaZooKeeperServer(Reconfigurer reconfigurer, ZookeeperServerConfig zookeeperServerConfig) { + peer = reconfigurer.startOrReconfigure(zookeeperServerConfig, this, () -> peer = new VespaQuorumPeer()); + } + + @Override + public void shutdown() { + peer.shutdown(Duration.ofMinutes(1)); + } + + @Override + public void start(Path configFilePath) { + peer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { + return true; + } + +} diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java new file mode 100644 index 00000000000..66742b0e05b --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaMtlsAuthenticationProvider.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.common.X509Exception; +import org.apache.zookeeper.data.Id; +import org.apache.zookeeper.server.ServerCnxn; +import org.apache.zookeeper.server.auth.AuthenticationProvider; +import org.apache.zookeeper.server.auth.X509AuthenticationProvider; + +import java.security.cert.X509Certificate; +import java.util.logging.Logger; + +/** + * A {@link AuthenticationProvider} to be used in combination with Vespa mTLS + * + * @author bjorncs + */ +public class VespaMtlsAuthenticationProvider extends X509AuthenticationProvider { + + private static final Logger log = Logger.getLogger(VespaMtlsAuthenticationProvider.class.getName()); + + public VespaMtlsAuthenticationProvider() throws X509Exception { super(null, null);} + + @Override + public KeeperException.Code handleAuthentication(ServerCnxn cnxn, byte[] authData) { + // Vespa's mTLS peer authorization rules are performed by the underlying trust manager implementation. + // The client is authorized once the SSL handshake has completed. + X509Certificate[] certificateChain = (X509Certificate[]) cnxn.getClientCertificateChain(); + if (certificateChain == null || certificateChain.length == 0) { + log.warning("Client not authenticated - should not be possible with clientAuth=NEED"); + return KeeperException.Code.AUTHFAILED; + } + X509Certificate certificate = certificateChain[0]; + cnxn.addAuthInfo(new Id(getScheme(), certificate.getSubjectX500Principal().getName())); + return KeeperException.Code.OK; + } + + @Override public String getScheme() { return "x509"; } + +} diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java new file mode 100644 index 00000000000..47ec03367c1 --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaQuorumPeer.java @@ -0,0 +1,60 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.protect.Process; +import org.apache.zookeeper.server.admin.AdminServer; +import org.apache.zookeeper.server.quorum.QuorumPeerConfig; +import org.apache.zookeeper.server.quorum.QuorumPeerMain; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Starts/stops a ZooKeeper server. Extends QuorumPeerMain to be able to call initializeAndRun() and wraps + * exceptions so it can be used by code that does not depend on ZooKeeper. + * + * @author hmusum + */ +class VespaQuorumPeer extends QuorumPeerMain implements QuorumPeer { + + private static final Logger log = java.util.logging.Logger.getLogger(VespaQuorumPeer.class.getName()); + + @Override + public void start(Path path) { + initializeAndRun(new String[]{ path.toFile().getAbsolutePath()}); + } + + @Override + public void shutdown(Duration timeout) { + if (quorumPeer != null) { + log.log(Level.FINE, "Shutting down ZooKeeper server"); + try { + quorumPeer.shutdown(); + quorumPeer.join(timeout.toMillis()); // Wait for shutdown to complete + if (quorumPeer.isAlive()) + throw new IllegalStateException("Peer still alive after " + timeout); + } catch (RuntimeException | InterruptedException e) { + // If shutdown fails, we have no other option than forcing the JVM to stop and letting it be restarted. + // + // When a VespaZooKeeperServer component receives a new config, the container will try to start a new + // server with the new config, this will fail until the old server is deconstructed. If the old server + // fails to deconstruct/shut down, the new one will never start and if that happens forcing a restart is + // the better option. + Process.logAndDie("Failed to shut down ZooKeeper server properly, forcing shutdown", e); + } + } + } + + @Override + protected void initializeAndRun(String[] args) { + try { + super.initializeAndRun(args); + } catch (QuorumPeerConfig.ConfigException | IOException | AdminServer.AdminServerException e) { + throw new RuntimeException("Exception when initializing or running ZooKeeper server", e); + } + } + +} diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java new file mode 100644 index 00000000000..ae7bf8d84f5 --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperAdminImpl.java @@ -0,0 +1,57 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @author hmusum + */ +@SuppressWarnings("unused") // Created by injection +public class VespaZooKeeperAdminImpl implements VespaZooKeeperAdmin { + + private static final Logger log = java.util.logging.Logger.getLogger(VespaZooKeeperAdminImpl.class.getName()); + + @Override + public void reconfigure(String connectionSpec, String servers) throws ReconfigException { + try (ZooKeeperAdmin zooKeeperAdmin = createAdmin(connectionSpec)) { + long fromConfig = -1; + // Using string parameters because the List variant of reconfigure fails to join empty lists (observed on 3.5.6, fixed in 3.7.0). + log.log(Level.INFO, "Applying ZooKeeper config: " + servers); + byte[] appliedConfig = zooKeeperAdmin.reconfigure(null, null, servers, fromConfig, null); + log.log(Level.INFO, "Applied ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + + // Verify by issuing a write operation; this is only accepted once new quorum is obtained. + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = zooKeeperAdmin.create("/reconfigure-dummy-node", new byte[0], acl, CreateMode.EPHEMERAL_SEQUENTIAL); + zooKeeperAdmin.delete(node, -1); + + log.log(Level.INFO, "Verified ZooKeeper config: " + new String(appliedConfig, StandardCharsets.UTF_8)); + } + catch ( KeeperException.ReconfigInProgress + | KeeperException.ConnectionLossException + | KeeperException.NewConfigNoQuorum e) { + throw new ReconfigException(e); + } + catch (KeeperException | IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + private ZooKeeperAdmin createAdmin(String connectionSpec) throws IOException { + return new ZooKeeperAdmin(connectionSpec, (int) sessionTimeout().toMillis(), + (event) -> log.log(Level.INFO, event.toString()), new ZkClientConfigBuilder().toConfig()); + } + +} + diff --git a/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java new file mode 100644 index 00000000000..48f95d28910 --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/com/yahoo/vespa/zookeeper/VespaZooKeeperServerImpl.java @@ -0,0 +1,47 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeeper; + +import com.yahoo.component.annotation.Inject; +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.component.AbstractComponent; + +import java.nio.file.Path; +import java.time.Duration; + +/** + * @author Ulf Lilleengen + * @author Harald Musum + */ +public class VespaZooKeeperServerImpl extends AbstractComponent implements VespaZooKeeperServer { + + private final VespaQuorumPeer peer; + private final ZooKeeperRunner runner; + + @Inject + public VespaZooKeeperServerImpl(ZookeeperServerConfig zookeeperServerConfig) { + this.peer = new VespaQuorumPeer(); + this.runner = new ZooKeeperRunner(zookeeperServerConfig, this); + } + + @Override + public void deconstruct() { + runner.shutdown(); + super.deconstruct(); + } + + @Override + public void shutdown() { + peer.shutdown(Duration.ofMinutes(1)); + } + + @Override + public void start(Path configFilePath) { + peer.start(configFilePath); + } + + @Override + public boolean reconfigurable() { + return false; + } + +} diff --git a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/NetUtils.java b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/NetUtils.java new file mode 100644 index 00000000000..33ec9b1303a --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/common/NetUtils.java @@ -0,0 +1,94 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.zookeeper.common; + +import java.net.Inet6Address; +import java.net.InetAddress; +import java.net.InetSocketAddress; + +/** + * This class contains common utilities for netstuff. Like printing IPv6 literals correctly + */ +public class NetUtils { + + // Note: Changed from original to use hostname from InetSocketAddress if there exists one + public static String formatInetAddr(InetSocketAddress addr) { + String hostName = addr.getHostName(); + if (hostName != null) { + return String.format("%s:%s", hostName, addr.getPort()); + } + + InetAddress ia = addr.getAddress(); + + if (ia == null) { + return String.format("%s:%s", addr.getHostString(), addr.getPort()); + } + if (ia instanceof Inet6Address) { + return String.format("[%s]:%s", ia.getHostAddress(), addr.getPort()); + } else { + return String.format("%s:%s", ia.getHostAddress(), addr.getPort()); + } + } + + /** + * Separates host and port from given host port string if host port string is enclosed + * within square bracket. + * + * @param hostPort host port string + * @return String[]{host, port} if host port string is host:port + * or String[] {host, port:port} if host port string is host:port:port + * or String[] {host} if host port string is host + * or String[]{} if not a ipv6 host port string. + */ + public static String[] getIPV6HostAndPort(String hostPort) { + if (hostPort.startsWith("[")) { + int i = hostPort.lastIndexOf(']'); + if (i < 0) { + throw new IllegalArgumentException( + hostPort + " starts with '[' but has no matching ']'"); + } + String host = hostPort.substring(1, i); + if (host.isEmpty()) { + throw new IllegalArgumentException(host + " is empty."); + } + if (hostPort.length() > i + 1) { + return getHostPort(hostPort, i, host); + } + return new String[] { host }; + } else { + //Not an IPV6 host port string + return new String[] {}; + } + } + + private static String[] getHostPort(String hostPort, int indexOfClosingBracket, String host) { + // [127::1]:2181 , check separator : exits + if (hostPort.charAt(indexOfClosingBracket + 1) != ':') { + throw new IllegalArgumentException(hostPort + " does not have : after ]"); + } + // [127::1]: scenario + if (indexOfClosingBracket + 2 == hostPort.length()) { + throw new IllegalArgumentException(hostPort + " doesn't have a port after colon."); + } + //do not include + String port = hostPort.substring(indexOfClosingBracket + 2); + return new String[] { host, port }; + } +} diff --git a/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java new file mode 100644 index 00000000000..fdfe0fe8467 --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/main/java/org/apache/zookeeper/server/VespaNettyServerCnxnFactory.java @@ -0,0 +1,37 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package org.apache.zookeeper.server; + +import com.yahoo.vespa.zookeeper.Configurator; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.logging.Logger; + +/** + * Overrides secure setting with value from {@link Configurator}. + * Workaround for incorrect handling of clientSecurePort in combination with ZooKeeper Dynamic Reconfiguration in 3.6.2 + * See https://issues.apache.org/jira/browse/ZOOKEEPER-3577. + * + * Using package {@link org.apache.zookeeper.server} as {@link NettyServerCnxnFactory#NettyServerCnxnFactory()} is package-private. + * + * @author bjorncs + */ +public class VespaNettyServerCnxnFactory extends NettyServerCnxnFactory { + + private static final Logger log = Logger.getLogger(VespaNettyServerCnxnFactory.class.getName()); + + private final boolean isSecure; + + public VespaNettyServerCnxnFactory() { + super(); + this.isSecure = Configurator.VespaNettyServerCnxnFactory_isSecure; + boolean portUnificationEnabled = Boolean.getBoolean(NettyServerCnxnFactory.PORT_UNIFICATION_KEY); + log.info(String.format("For %h: isSecure=%b, portUnification=%b", this, isSecure, portUnificationEnabled)); + } + + @Override + public void configure(InetSocketAddress addr, int maxClientCnxns, int backlog, boolean secure) throws IOException { + log.info(String.format("For %h: configured() invoked with parameter 'secure'=%b, overridden to %b", this, secure, isSecure)); + super.configure(addr, maxClientCnxns, backlog, isSecure); + } +} diff --git a/zookeeper-server/zookeeper-server/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java b/zookeeper-server/zookeeper-server/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java new file mode 100644 index 00000000000..db643d76e0d --- /dev/null +++ b/zookeeper-server/zookeeper-server/src/test/java/com/yahoo/vespa/zookeper/VespaZooKeeperTest.java @@ -0,0 +1,259 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.zookeper; + +import com.yahoo.cloud.config.ZookeeperServerConfig; +import com.yahoo.net.HostName; +import com.yahoo.vespa.zookeeper.ReconfigurableVespaZooKeeperServer; +import com.yahoo.vespa.zookeeper.Reconfigurer; +import com.yahoo.vespa.zookeeper.VespaZooKeeperAdminImpl; +import com.yahoo.vespa.zookeeper.client.ZkClientConfigBuilder; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.ZooDefs; +import org.apache.zookeeper.admin.ZooKeeperAdmin; +import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; +import org.junit.Ignore; +import org.junit.Test; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.ServerSocket; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.Phaser; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.IntStream; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.concurrent.TimeUnit.SECONDS; +import static java.util.stream.Collectors.toList; +import static org.junit.Assert.assertEquals; + +public class VespaZooKeeperTest { + + static final Path tempDirRoot = getTmpDir(); + static final List<Integer> ports = new ArrayList<>(); + + /** + * Performs dynamic reconfiguration of ZooKeeper servers. + * + * First, a cluster of 3 servers is set up, and some data is written to it. + * Then, 3 new servers are added, and the first 3 marked for retirement; + * this should force the quorum to move the 3 new servers, but not disconnect the old ones. + * Next, the old servers are removed. + * Then, the cluster is reduced to size 1. + * Finally, the cluster grows to size 3 again. + * + * Throughout all of this, quorum should remain, and the data should remain the same. + */ + @Test(timeout = 120_000) + @Ignore // Unstable, some ZK server keeps resetting connections sometimes. + public void testReconfiguration() throws ExecutionException, InterruptedException, IOException, KeeperException, TimeoutException { + List<ZooKeeper> keepers = new ArrayList<>(); + for (int i = 0; i < 8; i++) keepers.add(new ZooKeeper()); + for (int i = 0; i < 8; i++) keepers.get(i).run(); + + // Start the first three servers. + List<ZookeeperServerConfig> configs = getConfigs(0, 0, 3, 0); + for (int i = 0; i < 3; i++) keepers.get(i).config = configs.get(i); + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for all servers to be up and running. + for (int i = 0; i < 3; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Write data to verify later. + String path = writeData(configs.get(0)); + + // Let three new servers join, causing the three older ones to retire and leave the ensemble. + configs = getConfigs(0, 3, 3, 3); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // The existing servers can't reconfigure and leave before the joiners are up. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Wait for new quorum to be established. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + // Old servers are removed. + configs = getConfigs(3, 0, 3, 0); + for (int i = 0; i < 6; i++) keepers.get(i).config = configs.get(i); + // Old servers shut down, while the newer servers remain. + for (int i = 0; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Ensure old servers shut down properly. + for (int i = 0; i < 3; i++) keepers.get(i).await(); + // Ensure new servers have reconfigured. + for (int i = 3; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + + // Verify written data is preserved. + verifyData(path, configs.get(3)); + + + // Cluster shrinks to a single server. + configs = getConfigs(5, 0, 1, 0); + for (int i = 3; i < 6; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 6; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // We let the remaining server reconfigure the others out before they die. + for (int i = 3; i < 5; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 3; i < 5; i++) keepers.get(i).await(); + verifyData(path, configs.get(5)); + + // Cluster grows to 3 servers again. + configs = getConfigs(5, 0, 3, 2); + for (int i = 5; i < 8; i++) keepers.get(i).config = configs.get(i); + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + // Wait for the joiners. + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + verifyData(path, configs.get(7)); + + // Let the remaining servers terminate. + for (int i = 5; i < 8; i++) keepers.get(i).config = null; + for (int i = 5; i < 8; i++) keepers.get(i).phaser.arriveAndAwaitAdvance(); + for (int i = 5; i < 8; i++) keepers.get(i).await(); + } + + static String writeData(ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + try (ZooKeeperAdmin admin = createAdmin(config)) { + List<ACL> acl = ZooDefs.Ids.OPEN_ACL_UNSAFE; + String node = admin.create("/test-node", "hi".getBytes(UTF_8), acl, CreateMode.EPHEMERAL_SEQUENTIAL); + String read = new String(admin.getData(node, false, new Stat()), UTF_8); + assertEquals("hi", read); + return node; + } + } + + static void verifyData(String path, ZookeeperServerConfig config) throws IOException, InterruptedException, KeeperException { + for (int i = 0; i < 10; i++) { + try (ZooKeeperAdmin admin = createAdmin(config)) { + assertEquals("hi", new String(admin.getData(path, false, new Stat()), UTF_8)); + return; + } + catch (KeeperException.ConnectionLossException e) { + e.printStackTrace(); + Thread.sleep(10 << i); + } + } + } + + static ZooKeeperAdmin createAdmin(ZookeeperServerConfig config) throws IOException { + return new ZooKeeperAdmin(HostName.getLocalhost() + ":" + config.clientPort(), + 10_000, + System.err::println, + new ZkClientConfigBuilder().toConfig()); + } + + static class ZooKeeper { + + final ExecutorService executor = Executors.newSingleThreadExecutor(); + final Phaser phaser = new Phaser(2); + final AtomicReference<Future<?>> future = new AtomicReference<>(); + ZookeeperServerConfig config; + + void run() { + future.set(executor.submit(() -> { + Reconfigurer reconfigurer = new Reconfigurer(new VespaZooKeeperAdminImpl()); + phaser.arriveAndAwaitAdvance(); + while (config != null) { + new ReconfigurableVespaZooKeeperServer(reconfigurer, config); + phaser.arriveAndAwaitAdvance(); // server is now up, let test thread sync here + phaser.arriveAndAwaitAdvance(); // wait before reconfig/teardown to let test thread do stuff + } + reconfigurer.deconstruct(); + })); + } + + void await() throws ExecutionException, InterruptedException, TimeoutException { + future.get().get(30, SECONDS); + } + } + + static List<ZookeeperServerConfig> getConfigs(int removed, int retired, int active, int joining) { + return IntStream.rangeClosed(1, removed + retired + active) + .mapToObj(id -> getConfig(removed, retired, active, joining, id)) + .collect(toList()); + } + + // Config for server #id among retired + active servers, of which the last may be joining, and with offset removed. + static ZookeeperServerConfig getConfig(int removed, int retired, int active, int joining, int id) { + if (id <= removed) + return null; + + Path tempDir = tempDirRoot.resolve("zookeeper-" + id); + return new ZookeeperServerConfig.Builder() + .clientPort(getPorts(id).get(0)) + .dataDir(tempDir.toString()) + .zooKeeperConfigFile(tempDir.resolve("zookeeper.cfg").toString()) + .myid(id) + .myidFile(tempDir.resolve("myid").toString()) + .dynamicReconfiguration(true) + .server(IntStream.rangeClosed(removed + 1, removed + retired + active) + .mapToObj(i -> new ZookeeperServerConfig.Server.Builder() + .id(i) + .clientPort(getPorts(i).get(0)) + .electionPort(getPorts(i).get(1)) + .quorumPort(getPorts(i).get(2)) + .hostname("localhost") + .joining(i - removed > retired + active - joining) + .retired(i - removed <= retired)) + .collect(toList())) + .build(); + } + + static List<Integer> getPorts(int id) { + if (ports.size() < id * 3) { + int previousPort; + if (ports.isEmpty()) { + String[] version = System.getProperty("zk-version").split("\\."); + int versionPortOffset = 0; + for (String part : version) + versionPortOffset = 32 * (versionPortOffset + Integer.parseInt(part)); + previousPort = 20000 + versionPortOffset % 30000; + } + else + previousPort = ports.get(ports.size() - 1); + + for (int i = 0; i < 3; i++) + ports.add(previousPort = nextPort(previousPort)); + } + return ports.subList(id * 3 - 3, id * 3); + } + + static int nextPort(int previousPort) { + for (int j = 1; j <= 30000; j++) { + int port = (previousPort + j); + while (port > 50000) + port -= 30000; + + try (ServerSocket socket = new ServerSocket(port)) { + return socket.getLocalPort(); + } + catch (IOException e) { + System.err.println("Could not bind port " + port + ": " + e); + } + } + throw new RuntimeException("No free ports"); + } + + static Path getTmpDir() { + try { + Path tempDir = Files.createTempDirectory(Paths.get(System.getProperty("java.io.tmpdir")), "vespa-zk-test"); + tempDir.toFile().deleteOnExit(); + return tempDir.toAbsolutePath(); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} |